pax_global_header00006660000000000000000000000064151655410060014515gustar00rootroot0000000000000052 comment=b1fb5fac3440ae2f3eb54e2da8f4f09e11666ce5 just-buildsystem-justbuild-b1fb5fa/000077500000000000000000000000001516554100600176175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/.clang-format000066400000000000000000000014701516554100600221740ustar00rootroot00000000000000--- Language: Cpp Standard: c++20 BasedOnStyle: Google AccessModifierOffset: -2 AllowAllParametersOfDeclarationOnNextLine: false AllowShortFunctionsOnASingleLine: Inline AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false BinPackArguments: false BinPackParameters: false BraceWrapping: BeforeElse: true IndentBraces: false SplitEmptyFunction: true SplitEmptyNamespace: true SplitEmptyRecord: true BreakBeforeBraces: Custom ColumnLimit: 80 DerivePointerAlignment: false IncludeCategories: # The base style already correctly handles system includes # C-style third-party includes - Regex: '^<.*\.(h|hpp)>' Priority: 10 # General external and project includes - Regex: '^".*"' Priority: 20 IndentWidth: 4 KeepEmptyLinesAtTheStartOfBlocks: true PenaltyBreakString: 100 ... just-buildsystem-justbuild-b1fb5fa/.clang-tidy000066400000000000000000000102231516554100600216510ustar00rootroot00000000000000# Here is an explanation for why some of the checks are disabled: # bugprone-easily-swappable-parameters: This check would require significant # refactoring effort. # bugprone-unchecked-optional-access: Too many false positives. For example, # an explicit comparison with std::nullopt isn't considered a check. # cppcoreguidelines-avoid-const-or-ref-data-members: We believe, ref data # members are a good way to express ownership, and const data members improve # readability. # cppcoreguidelines-rvalue-reference-param-not-moved: Too many false positives, # especially with partial moves from STL containers. # misc-const-correctness: Too many false positives, especially with STL # containers. # misc-include-cleaner: There is no way for symbol mapping. For example, this # check requires to delete the "gsl/gsl" header, but at the same time asks to # include one for gsl::not_null. # misc-use-anonymous-namespace: This check would require significant refactoring # effort, but wouldn't improve readability equally. # misc-no-recursion: There are legitimate uses for us: we use recursion for # trees a lot. # modernize-return-braced-init-list: We think removing typenames and using only # braced-init can hurt readability. # performance-avoid-endl: There are too many legitimate uses of std::endl for # us. # readability-function-cognitive-complexity: This check would get triggered by # most uses of the catch2 test library. # readability-identifier-length: We would like to enable this check, but it # would require significant refactoring effort. # readability-redundant-member-init: Those are not redundant for structs, but # clang-tidy keeps reporting this as an error. FormatStyle: Google Checks: >- *,-abseil-*,-altera-*,-android-*,-boost-*,-cert-*,-darwin-*,-fuchsia-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-zircon-*, -bugprone-easily-swappable-parameters,-bugprone-unchecked-optional-access, -clang-analyzer-cplusplus.NewDeleteLeaks, -clang-diagnostic-unused-command-line-argument, -concurrency-mt-unsafe, -cppcoreguidelines-avoid-const-or-ref-data-members,-cppcoreguidelines-rvalue-reference-param-not-moved, -misc-const-correctness,-misc-include-cleaner,-misc-use-anonymous-namespace,-misc-no-recursion, -modernize-return-braced-init-list, -performance-avoid-endl, -readability-function-cognitive-complexity,-readability-identifier-length,-readability-redundant-member-init WarningsAsErrors: '*' CheckOptions: - { key: misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic, value: '1' } - { key: readability-identifier-naming.ClassCase, value: CamelCase } - { key: readability-identifier-naming.EnumCase, value: CamelCase } - { key: readability-identifier-naming.StructCase, value: CamelCase } - { key: readability-identifier-naming.TypeTemplateParameterCase, value: CamelCase } - { key: readability-identifier-naming.ConstexprVariableCase, value: CamelCase } - { key: readability-identifier-naming.ConstexprVariablePrefix, value: k } - { key: readability-identifier-naming.GlobalConstantCase, value: CamelCase } - { key: readability-identifier-naming.GlobalConstantPrefix, value: k } - { key: readability-identifier-naming.MemberConstantCase, value: CamelCase } - { key: readability-identifier-naming.MemberConstantPrefix, value: k } - { key: readability-identifier-naming.StaticConstantCase, value: CamelCase } - { key: readability-identifier-naming.StaticConstantPrefix, value: k } - { key: readability-identifier-naming.ValueTemplateParameterCase, value: CamelCase } - { key: readability-identifier-naming.ValueTemplateParameterPrefix, value: k } - { key: readability-identifier-naming.VariableCase, value: lower_case } - { key: readability-identifier-naming.ClassMemberCase, value: lower_case } - { key: readability-identifier-naming.PrivateMemberSuffix, value: _ } - { key: readability-identifier-naming.ProtectedMemberSuffix, value: _ } just-buildsystem-justbuild-b1fb5fa/CHANGELOG.md000066400000000000000000000666311516554100600214440ustar00rootroot00000000000000## Release `1.6.5` (2026-04-08) Bug fixes on top of `1.6.4`. ### Fixes - `just execute` now allows, as demanded by the protocol, instance names having an arbitrary number of segments, possibly none. - `just execute` now adds at startup the empty blob to CAS thus fulfilling the assumption by some build tools that the empty blob can always be referenced without ever uploading it. - The value of the `instance_name` for the remote-execution endpoint can be set now using the option `--remote-instance-name`, instead of using the hard-coded value `"remote-execution"`. Also, the default value has been set to `""` which is the more common default; use said option to restore previous behaviour. ## Release `1.6.4` (2026-02-14) Bug fixes on top of `1.6.3`. ### Fixes - Overlaying the empty list of trees now returns the empty tree instead of causing a segmentation fault. - `just-mr` no longer crashes if the empty string is specified as path for a `"file"` repository; instead it treats it as `"."`. - Improved documentation. ## Release `1.6.3` (2025-08-11) Bug fixes on top of `1.6.2`. ### Fixes - In compatible mode, `just` now properly handles symbolic links that occur in output directories or as explicit outputs. The problem was that the default remote-execution protocol (which is used in compatible mode) does not require the remote endpoint to store symbolic links explicitly as blobs in CAS, while `just` internally does exactly that and is assuming symbolic links being available in CAS. This fix now uploads any symbolic link that is received over the network (e.g., as part of a Directory message) in compatible mode as separate blob to the remote CAS. ## Release `1.6.2` (2025-07-30) Bug fixes on top of `1.6.1`. ### Fixes - `just serve` can now also work with a compatible remote-execution service that relies on the length field of digests being set correctly (as requested by the protocol). Previously, some blobs were requested by correct hash but with size set to `0`. To keep everything backwards compatible (both, when only `serve` or only `just` gets updated), the `ServeTargetRequest` of the internal `serve` protocol was extended by a new field where the client can send a list of full blob digests asking the server to download those before handling the request. - If `serve` reports an error, the full digest of the error-log blob (including size) is shown to the user and not only the hash. This allows inspection of the error log, even if a strict compatible remote execution is used. ## Release `1.6.1` (2025-07-16) Bug fixes on top of `1.6.0`. ### Fixes - `just` now follows the protocol for blob splitting/splicing that was standardized in https://github.com/bazelbuild/remote-apis/pull/282 Before, this used to be `just`-specific extension. - Fixed missing mandatory locking point of the Git cache in a `just serve` service. - `just-mr` now correctly maintains also the Git cache lock when calling `just` if repository configuration was involved, preventing any unwanted intermediary repository garbage collection. - The invocation server now correctly reports the total number of uncached actions. - Allow colons present in remote execution properties. - Improvements to the documentation. ### Note for package maintainers The dependency on the remote-execution API has been changed to require a newer version. As this is often provided as a separate source archive, this might require additional changes. ## Release `1.6.0` (2025-06-27) A feature release on top of `1.5.0`, backwards compatible. ### New features - `just-mr` now supports logging of each invocation by setting an appropriate configuration in the rc-file. Together with the newly-added option `--profile` of `just` this can be used to gather statistics on build times, cache hit rates, as well as their evolution over time. - Computing a tree as overlays of other trees was added as a new in-memory action. - The expression language has been extended to contain new built-in functions `"zip_with"`, `"zip_map"`. ### Other changes - The exit code 1 now strictly refers to build failure due to a failing build action. Syntactical errors invoking the tool, as well as errors during analysis now return separate exit codes. ### Fixes - `just execute` and `just serve` now create their pid and info files atomically; so waiting processes can assume the content to be available as soon as the requested file appears on the file system. - `just serve` now fetches trees from remote execution in parallel and through its local CAS; this fixes a performance issue. - A bug was fixed that could cause the number of threads being the square of what was specified during backing up artifacts of export targets after build. - `just-mr` now also considers computed roots (as no-op) when reporting progress. - The "generic" rule now properly detects staging conflicts, taking the full inputs into account (and not only the runfiles). - Illegitimate symlinks in explicit source-tree references are now rejected reliably. - An incorrect error handling in the evaluation of computed roots was fixed. - `just execute`: Symlinks to directories and files are properly distinguished as requested by the remote-execution protocol. - Various improvements of the documentation. ### Note for package maintainers Any patching that used to patch `etc/repos.json` should now patch `etc/repos.in.json`. Background: to allow linting with well-defined dependencies, for the newly-added `"lint"` repository additional dependencies are pulled in via `just-lock` to bootstrap the correct versions of those tools; the local, manually-edited repository configuration `etc/repos.in.json` still contains everything needed for building and testing. So, to avoid accidentally trying to fetch more than is absolutely needed, `bin/bootstrap.py` was changed to use the original, manually-maintained `etc/repos.in.json`. ### Changes since `1.6.0~beta1` - The flexible variables of the bundled `ssl` description have been fixed, fixing a bootstrap issue in bundled (i.e., non-package) build. - The setup of one test has been fixed. - Various improvements to the documentation. ## Release `1.6.0~beta1` (2025-06-24) First beta release for the upcoming `1.6.0` release; see release notes there. ## Release `1.5.0` (2025-03-06) A feature release on top of `1.4.0`, backwards compatible. ### Major new features - Added two new root types: `"computed"` and `"tree structure"`. This allows to define roots as the artifacts of an export target of an earlier-defined content-fixed repository, as well as the underlying tree structure of an earlier-defined content-fixed root. Both new root types are themselves content fixed. ### New features - A new tool `just-lock` has been added that allows to define `just-mr` repository configurations out of an abstract configuration defining base repositories and a sequence of imports. - An option `-p` was added to the building subcommands to show the unique artifact (if there is precisely one) on stdout. - The checkout-locations file now additionally allows to specify extra environment variables to inherit. - `just add-to-cas` now supports the `--resolve-special` option, which defines how special entries (e.g., symlinks) are to be handled when adding directories to CAS. - `just serve` accepts a new subkey `"client address"` for the key `"execution endpoint"` in the configuration file. It informs the `serve` instance that the client will access the remote-execution endpoint via the given `"client address"` and not using the one listed in the subkey `"address"`. This feature allows to position `just serve` next to the remote-execution endpoint behind some form of address translation. ### Fixes - `just-import-git` now correctly inherits pragmas for imported file-type repositories during description rewrite. - `just-mr` repository garbage collection now properly removes no longer needed directories. - The "generic" rule now properly detects staging conflicts. - Fixes ensuring proper pointer life time and access check. - A race condition in the use of `libgit2` was fixed that could result in a segmentation fault. - Git operations are now properly locked against each other, also between processes where necessary. - The Git cache root repository on a `just serve` endpoint is now ensured to always exist and be initialized before being operated on. - `just install-cas` correctly exits with non-zero exit code on failure, also if installation to stdout is requested. - `just traverse` now exits unconditionally after traversal, also in case of failure. - `just-mr` properly enforces that repository `subdir` entries are non-upwards relative paths. - The local api correctly handles not-found blobs, even in the absence of a local git api. - For remote execution, the server capability `max_batch_total_size_bytes` is now correctly honored, if announced by the server. - Missing entries in the documentation have been added. ### Changes since `1.5.0~beta2` - Fixed how `just-import-git` and `just-lock` handle the transitively implied base repositories of computed roots; the lack of properly handling indirections led to crashes even if computed roots where not used at all. - A case was fixed where special entries where not ignored properly, even though this was requested. - Unnecessary verbosity reduced. - Updated dependencies. - Documentation extended. ## Release `1.5.0~beta2` (2025-02-28) Second beta release for the upcoming `1.5.0` release; see release notes there. ### Changes since `1.5.0~beta1` - New configuration option `"client address"` for `just serve`. - `just-lock` now fetches and clones repositories in parallel. - Blob content is not any more kept in memory unnecessarily at various places. - Various internal clean up of the code base. ## Release `1.5.0~beta1` (2025-02-24) First beta release for the upcoming `1.5.0` release; see release notes there. ## Release `1.4.0` (2024-11-04) A feature release on top of `1.3.0`, backwards compatible with respect to rule language, build description, repository description, and wire protocols. However, the internal representation in local build root has changed; it is therefore recommended to remove the local build root on upgrade. ### New features - `just serve` now also works together with a compatible remote-execution endpoint. This uses an extended version of the serve protocol, so both, `just-mr` and `just serve` need to be at the new version. - User-defined rules, as well as the built-in rule `"generic"` can now specify a subdirectory in which an action is to be executed. - `just-mr` now supports garbage collection for repository roots via the `gc-repo` subcommand. This follows the same two-generation approach as garbage collection for the cache-CAS pair; in other words, everything is cleaned up that was not used since the last call to `gc-repo`. To accommodate this, the layout in the local build root had to be changed. The directory `git` as well as `*-map` directories are now located in the subdirectory `repositories/generation-0`. On upgrade those have to be manually moved there if they should be continued to be used; removing the whole local build root is, of course, also a valid upgrade path, however losing the whole cache. Not doing anything on upgrade will not lead to an inconsistent state; however, the directories at the old location will not be used anymore while still using disk space. - The expression language has been extended to contain quote and quasi-quote expressions, as well as new built-in functions `"from_subdir"`, `"nub_left"`. ### Fixes - The built-in rule `"generic"` now properly enforces that the obtained outputs form a well-formed artifact stage; a conflicting arrangement of artifacts was possible beforehand. - The built-in expression functions `"join"` and `"join_cmd"` now properly enforce that the argument is a list of strings. So far, they used to accept a single string, treating it as a singleton list. - A bug was fixed that cased `just serve` to fail with an internal error when building against ignore-special roots. - `just` now accurately reports internal errors that occurred on the serve endpoint. - Target-level cache entries are only written if all export targets depended upon are also written to or found in cache; previously, it was assumed that all export targets not analysed locally were local cache hits, an assumption that no longer holds in the presence of serve endpoints. This fixes a cache consistency problem if the same remote-execution endpoint is used both, with and without a serve endpoint. - A race condition in reconstructing executables from large CAS has been removed that could lead to an open file descriptor being kept alive for too long, resulting EBUSY failures of actions using this binary. - Internal code clean up, reducing memory footprint, in particular for simultaneous upload of a large number of blobs. - Avoidance of duplicate requests and performance improvements when synchronizing artifacts with another CAS. - Dependencies have been updated to also build with gcc 14. - Portability improvements of the code by not relying on implementation details of the compiler. - Local execution no longer has the requirement that there exist no more files with identical content than the hardlink limit of the underlying file system. - Inside action descriptions, paths are always normalized; this improves compatibility with existing remote-execution implementations. - The size of large object entries has been reduced. The cache and CAS must be cleaned up since stable versions before `1.4.0` cannot use the new format. - The way of storing intermediate keys of the action cache has been changed. The cache must be cleaned up since stable versions before `1.4.0` cannot use the new format. - Various improvements to the tests: dispatching of the summary action is now possible, tests are independent of a .just-mrrc file the user might have in their home directory - Various improvements of the documentation. ## Release `1.4.0~beta1` (2024-10-30) First beta release for the upcoming `1.4.0` release; see release notes there. ## Release `1.3.0` (2024-05-08) A feature release on top of `1.2.0`, backwards compatible. ### Major new features - New subcommand `just serve` to start a target-level caching service, as described in the corresponding design document. - `just-mr` is able to back up and retrieve distribution files from a remote execution endpoint. This simplifies usage in an environment with restricted internet access. - `just execute` now supports blob splitting as new RPC call. `just install` uses this call to reduce traffic if the remote-execution endpoint supports blob splitting and the `--remember` option is given. In this way, traffic from the remote-execution endpoint can be reduced when subsequently installing artifacts with only small local differences. ### Other changes - New script `just-deduplicate-repos` to avoid blow up of the `repos.json` in the case of chained imports with common dependencies. - New subcommand `add-to-cas` to add files and directories to the local CAS and optionally also copy them to the remote-execution endpoint. - The built-in `"generic"` rule now supports an argument `"sh -c"`, allowing to specify the invocation of the shell (defaulting to `["sh", "-c"]`). - `just describe` also shows the values of the implicit dependencies. - `just-mr` supports a new form of root, called `"foreign file"`. - When `just-mr` executes the action to generate the desired tree of a `"git tree"` repository, it can be specified that certain variables of the environment can be inherited. - The just-mr rc file now supports a field `"rc files"` to include other rc files given by location objects; in particular, it is possible to include rc files committed to the workspace. - Support for fetching archives from FTP and TFTP was added to `just-mr` if it was built with bundled curl. For package builds, libcurl has enabled whatever the distro considers suitable. - The `gc` subcommand supports an option `--no-rotate` to carry out only local clean up. Part of that local clean up, that is also done as part of a full `gc`, is splitting large files. Note that stable versions before `1.3.0` cannot use those split files. Hence a downgrade after a `gc` with `1.3.0` (or higher) requires cleaning of cache and CAS. - The expression language has been extended and, in particular, allows indexed access to an array (basically using it as a tuple) and a generic form of assertion (to report user errors). - The `analyse` subcommand supports a new flag `--dump-result` to dump the analysis result to a file or stdout (if `-` is given). ### Fixes - The cache key used for an export target is now based on the export target itself rather than that of the exported target. The latter could lead to spurious cache hits, but only in the case where the exported target was an explicit file reference, and a regular target with the same name existed as well. Where the new cache keys would overlap with the old ones, they would refer to the same configured targets. However, we used the fact that we changed the target cache key to also clean up the serialization format to only contain the JSON object describing repository, target, and effective configuration, instead of a singleton list containing this object. Therefore, old and new cache keys do not overlap at all. In particular, no special care has to be taken on upgrading or downgrading. However, old target-level cache entries will not be used leading potentially to rebuilding of some targets. - Garbage collection now honors the dependencies of target-level caches entries on one another. When upgrading in place, this only applies for target-level cache entries written initially after the upgrade. - The taintedness of `"configure"` targets is now propagated correctly in analysis. - It is no longer incorrectly assumed that every `git` URL not starting with `ssh://`, `http://`, nor `https://` is a file on the local disk. Now, only URLs starting with `/`, `./`, or `file://` are considered file URLs. File URLs, as well as URLs starting with `git://`, `http://`, or `https://`, are handled by `just-mr` using `libgit2`; for every other URL, `just-mr` shells out to `git` for fetching and the URL is passed to `git` unchanged. - Improved portability and update of the bundled dependencies. - Various minor improvements and typo fixes in the documentation. - Fixed a race condition in the task queue that could cause (with probability roughly 1e-5) a premature termination of the queue resulting in spurious analysis failures without explanation (despite "failed to analyse target"). - Fixed a race condition in an internal cache of `just execute` used for keeping track of running operations. - The built-in rule `"install"` now properly enforces that the resulting stage is well-formed, i.e., without tree conflicts. - Local execution and `just execute` now correctly create empty directories if they are part of the action's input. - Fixed overwrite of existing symlinks in the output directory when using subcommands `install` and `install-cas`. - The format for target-cache shards was changed to a canonical form. The new and old formats do not overlap, therefore the correctness of the builds is not affected. In particular, no special care has to be taken on upgrading or downgrading. However, some target-level cache entries will not be used leading potentially to rebuilding of some targets. - The expression `"disjoint_map_union"` did not verify disjointness in all cases; this is fixed now. - The command line option `"--remote-execution-property"` can be repeated multiple times to list all the properties, but only the last one was retained. This is fixed now. ### Changes since `1.3.0~beta1` - The `["CC/pkgconfig", "system_library"]` rule now propagates `ENV` correctly, fixing the build on systems where the default paths pulled in by `env` do not contain `cat`. - In case of a build failure, the description of the failing action in the error message is now more verbose, including the environment. - Various minor fixes in the documentation. ## Release `1.3.0~beta1` (2024-05-02) First beta release for the upcoming `1.3.0` release; see release notes there. ## Release `1.2.0` (2023-08-25) A feature release on top of `1.1.0`, backwards compatible. ### Major new features - Actions can now define additional execution properties and in that way chose a specific remote execution image, as well as a factor to scale the time out. This also applies to the built-in `generic` rule. Additionally, the remote-execution endpoint can be dispatched based on the remote-execution properties using the `--endpoint-configuration` argument. - Relative non-upwards symbolic links are now treated as first-class objects. This introduces a new artifact type and allows the free use of such symbolic links throughout the build process. - `just-mr` can now optionally resolve symlinks contained in archives. ### Other changes - `just-import-git` now supports an option `--plain` to import a repository without dependencies. - Minor changes to the layout of the local build root; in particular, left-over execution directories, as well as left-over temporary directories of `just-mr`, will eventually get cleaned up by garbage collection. - `just-mr` now supports unpacking tar archives compressed with bzip2, xz, lzip, and lzma. - The option `-P` of `build` and `install-cas` can be used to inspect parts of a tree. - `just-mr` now supports unpacking 7zip archives (with default compression) when provided as `"zip"` type repositories. - The configuration variable `COMPILER_FAMILY` is replaced by the more flexible `TOOLCHAIN_CONFIG`, an object which may contain the field `FAMILY`. From now on, this object is used to set the compiler family (e.g., for GNU, set `{"TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}`). ### Fixes - Removed potential uses of `malloc` between `fork` and `exec`. This removes the risk of deadlocks on certain combinations of `C++` standard library and `libc`. - The link flags for the final linking now can be set via the configuration variable `FINAL_LDFLAGS`; in particular, the stack size can easily be adapted. The default stack size is now set to 8M, removing an overflow on systems where the default stack size was significantly lower. - The man pages are now provided as markdown files, allowing to potentially reduce the build dependencies to more standard ones. - `just-mr` now correctly performs a forced add in order to stage all entries in a Git repository. Previously it was possible for entries to be skipped inadvertently in, e.g., imported archives if `gitignore` files were present. - Temporary files generated by `just execute` are now created inside the local build root. - `just install-cas` now correctly handles `--raw-tree` also for remote-execution endpoints. - `just install-cas` now, like `just install`, removes an existing destination file before installing instead of overwriting. - Only actions with exit code 0 that generated all required outputs are taken from cache, instead of all actions with exit code 0. This only affects remote execution, as purely local build didn't cache actions with incomplete outputs. ### Changes since `1.2.0~beta3` - Only actions with exit code 0 that generated all required outputs are taken from cache, instead of all actions with exit code 0. This only affects remote execution, as purely local build didn't cache actions with incomplete outputs. - Splitting off libraries from the main binary targets to simplify cherry-picking future fixes from the head development branch. - Improvements of the bundled dependency descriptions. - Update of documentation. ## Release `1.2.0~beta3` (2023-08-22) Third beta release for the upcoming `1.2.0` release; see release notes there. ### Changes since `1.2.0~beta2` - Update and clean up of bundled dependency descriptions - Improvement of documentation ## Release `1.2.0~beta2` (2023-08-18) Second beta release for the upcoming `1.2.0` release; see release notes there. ### Changes since `1.2.0~beta1` - Clean up of the internal build description of bundled dependencies. - Clean up of the internal rules, in particular renaming of implicit dependency targets. - Various documentation improvements. ## Release `1.2.0~beta1` (2023-08-16) First beta release for the upcoming `1.2.0` release; see release notes there. ## Release `1.1.0` (2023-05-19) A feature release on top of `1.0.0`, backwards compatible. ### Major new features - new subcommand `just execute` to start a single node execution service - New subcommand `just gc` to clean up no longer needed cache and CAS entries - `just` now supports authentication to remote execution via TLS and mutual TLS - `just-mr` is now available as C++ binary and supports fetching in parallel ### Important changes - The option `-D` now accumulates instead of ignoring all but the latest occurrence. This is an incompatible change of the command line, but not affecting the backwards compatibility of the build. - The option `-L` of `just-mr` now is an alternative name for option `--local-launcher` instead of `--checkout-locations`, and thus matching its meaning in `just`. This is an incompatible change of the command line, but not affecting the backwards compatibility of the build. ### Other changes - `just install` and `just install-cas` now have a new `--remember` option ensuring that the installed artifacts are also mirrored in local CAS - `just analyse` now supports a new option `--dump-export-targets` ### Note There is a regression in `libgit2` versions `1.6.1` up to and including `1.6.4` with a fix already committed upstream. This regression makes `just` unusable when built against those versions. Therefore, the third-party build description for `libgit2` is still for version `1.5.2`. ## Release `1.1.0~beta2` (2023-05-15) Second beta release for the upcoming `1.1.0` release; see release notes there. ### Changes since `1.1.0~beta1` - fix a race condition in our use of `libgit2` - a fix in the error handling of git trees - fixes to the third-party descriptions of our dependencies; in particular, the structure of the `export` targets is cleaned up. These changes should not affect package builds. - various minor fixes to documentation and tests ### Note There is a regression in `libgit2` versions `1.6.1` upto and including `1.6.4` with a fix already committed upstream. This regression makes `just` unusable when built against those versions. Therefore, the third-party build description for `libgit2` is still for version `1.5.2`. ## Release `1.1.0~beta1` (2023-04-28) First beta release for the upcoming `1.1.0` release; see release notes there. ## Release `1.0.0` (2022-12-12) Initial stable release. ### Important changes since `1.0.0~beta6` - built-in rule "tree" added - clean up of user-defined rules for C++ - various documentation improvements ## Release `1.0.0~beta6` (2022-10-16) ### Important changes since `1.0.0~beta5` - The "configure" built-in rule now evaluates "target". Also, a bug in the computation of the effective configuration was fixed. - Option `--dump-vars` added to `just analyse` - Rule fixes in propagating `ENV` - Launcher functionality added to `just-mr` - `just` now takes the lexicographically first repository as default if no main repository is specified ## Release `1.0.0~beta5` (2022-10-19) First public beta version. just-buildsystem-justbuild-b1fb5fa/CONTRIBUTING.md000066400000000000000000000061251516554100600220540ustar00rootroot00000000000000# Contributing ## Preparing changes Larger changes, as well as changes breaking backwards compatibility require a design document describing the planned change. It should carefully discuss the objective of the planned feature, possible alternatives, as well as the transition plan. This document has to be agreed upon and committed to the repository first. For all changes, remember to also update the documentation and add appropriate test coverage. For code to be accepted, all tests must pass; the global test suite is `["@", "just tests", "", "ALL"]`. Code is formatted with `clang-format` and linted with `clang-tidy`; the corresponding configuration files can be found in the top-level directory of this repository. The top-level lint target is `["@", "lint", "", ""]`. Formatting issues can be fixed by building and applying the patch `["@", "lint", "", "format.diff"]`; the script `bin/format-code.sh` does precisely this. *NOTE:* In order for everyone to use the same version of the linting tools, the `"lint"` repository [bootstraps](https://github.com/just-buildsystem/bootstrappable-toolchain) the required tools; the configuration variable `"TOOLCHAIN_CONFIG"` is honored. As a consequence, (transitively) depending on the `"lint"` repository pulls in quite some sources and linting the first time requires a significant amount of time to build the tools from first principles. This is also true when calling `bin/format-code.sh`. Target, rules, and expression files should be formatted using `bin/json-format.py`. The corresponding target is `["@", "format-json", "", ""]` which provides a diff to be applied in order to obtain well-formatted target, rules, and expression files; the script `bin/format-json.sh` computes and applies that diff. Changes should be organized as a patch series, i.e., as a sequence of small changes that are easy to review, but nevertheless self-contained in the sense that after each such change, the code builds and the tests pass; in particular, tests should be rebased to come after the implementation in the series to avoid spurious errors when doing `git bisect`. And the end of a patch series, there should be no "loose ends" like library functions added, but never used. ## Submitting patches Patches can be sent for review by creating a feature branch in a clone of the repository and creating a merge request on one of the `git`-hosting sites hosting this repository. Alternatively, patches can also be sent via email using `git format-patch`, `git send-email`. Please use the `--cover-letter` option and add the description of the patch series in the cover letter. When sending a revised version, e.g., because of review comments, please make the new cover letter `In-Reply-To:` the old over letter ## Review Each commit has to be reviewed by a core member of the project (a person with write access to the repository) that is not the author of the respective commit. The core member who did the review will also push the commits. Patch series go in as a whole or not at all. Changes are added fast-forward of the current `master` without a merge commit, rebasing the patches, if necessary. just-buildsystem-justbuild-b1fb5fa/INSTALL.md000066400000000000000000000171051516554100600212530ustar00rootroot00000000000000# Installing the `just` binary ## Building `just` using an older version of `just-mr` and `just` If an older installation of `just` is already available, `just` can simply be built by ```sh just-mr build ``` This is always guaranteed to work with the latest stable release of `just`. This will build `just` for Linux on the x86_64 architecture with a dynamic link dependency on glibc. ### Building `just` for other architectures First, make sure that the cross-compiler for the desired architecture is installed and working properly. For example, to build `just` for 64 bit ARM, specify `arm64` as the target architecture via the `-D` flag: ```sh just-mr build -D '{"TARGET_ARCH":"arm64"}' ``` The following table describes the most important supported configuration variables. The full list can be obtained via `just-mr describe`. |Variable|Supported Values|Default Value for `just`| |-|:-:|:-:| | OS | linux | linux | | ARCH | x86, x86_64, arm, arm64 | x86_64 | | HOST_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* | | TARGET_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* | | DEBUG | map, anything logically false | null | | TOOLCHAIN_CONFIG["FAMILY"] | gnu, clang, unknown | unknown | | TOOLCHAIN_CONFIG["BUILD_STATIC"] | true, false | false | Note that you can choose a different stack size for resulting binaries by adding `"-Wl,-z,stack-size="` to variable `"FINAL_LDFLAGS"` (which has to be a list of strings). ## Bootstrapping `just` It is also possible to build `just` without having an older binary, using the `bin/bootstrap.py` script. ### Bootstrapping compiling the dependencies from scratch If using bundled dependencies is acceptable, the only things required are a C++20 compiler with the libraries required by the language standard (note that, e.g., in Debian `libstdc++-10-dev` is not a dependency of `clang`), a C compiler, and a Python3 interpreter. By default, the bootstrap script uses `c++` as C++ compiler and `cc` as C compiler. If you also want the bootstrap script to download the dependencies itself, `wget` is required as well. In this case, the command is simply ```sh python3 ./bin/bootstrap.py ``` The script also takes optionally the following positional arguments (in the given order, i.e., specifying one argument requires the ones before to be present as well). - The directory of the source location (defaulting to the current working directory). Specifying that directory allows calling the script from a different location. It should be noted that the script is written in such a way that the source is not modified. - The scratch directory (defaulting to python's `tempfile.mkdtemp()`). The script assumes it can use that directory entirely on its own with no other processes interfering. The bootstrapped binary has path `out/bin/just` relative to that directory. - A directory where (some of) the archives of the dependencies are downloaded ahead of time (defaulting to `.distfiles` in the user's home directory). Whenever an archive is needed, it is first checked if a file with the basename of the URL exists in this directory and has the expected blob id (computed the same way as `git` does). Only if this is not the case, fetching from the network is attempted. Additionally, if the environment variable `DEBUG` is set, the second bootstrap phase is carried out sequentially rather than in parallel. Moreover, when constructing the build configuration, the scripts starts with the value of the environment variable `JUST_BUILD_CONF` instead of the empty object, if this variable is set. One configuration parameter is the build environment `ENV` that can be used to set an unusual value of `PATH`, e.g., ``` sh env JUST_BUILD_CONF='{"ENV": {"PATH": "/opt/toolchain/bin"}}' python3 ./bin/bootstrap.py ``` Additionally, if `SOURCE_DATE_EPOCH` is set in the build environment, it is forwarded to the build configuration as well. If, on the other hand, `CC` or `CXX` are set in the build configuration, those are also used for the initial steps of the bootstrap procedure. Remember that setting one of those variables also requires the `TOOLCHAIN_CONFIG["FAMILY"]` to ensure the proper flags are used (if in doubt, set to `"unknown"`). In any case, the resulting binary is self contained and can be moved to an appropriate location in `PATH`. ### Bootstrapping against preinstalled dependencies (package building) The main task is to ensure all the dependencies are available at sufficiently compatible versions. The full list of dependencies can be found in `etc/repos.in.json`. This file also specifies, in the `"local_path"` attribute of `"pkg_bootstrap"`, the location relative to `LOCALBASE` (typically `/usr` or `/usr/local`) that is taken as root for the logical repository of that dependency. As, however, library dependencies are taken via `pkg-config`, in most cases, setting this attribute should not be necessary. The target files for the dependencies can be found in `etc/import.pkgconfig`. Possibly different names to be used by `pkg-config` can be adapted there. If the environment variable `PKG_CONFIG_PATH` is set, the bootstrap script forwards it to the build so that `pkg-config` can pick up the correct files. The build command is the same (with the same positional arguments), however with the environment variable `PACKAGE` being present and `LOCALBASE` set accordingly. As package building requires a predictable location on where to pick up the resulting binary, you almost certainly want to set the scratch directory. ```sh env PACKAGE=YES LOCALBASE=/usr python3 ${SRCDIR}/bin/bootstrap.py ${SRCDIR} ${BUILDDIR} ``` If some dependencies should nevertheless be built from source (typically taking the necessary archives from a specified distdir) those can be specified in the `NON_LOCAL_DEPS` variable which, if set, has to contain a JSON list. # Installing `just-mr` In order to set up multi-repository configurations, usually the tool `just-mr` is used. It is also a useful launcher for `just`. This tool can be obtained by building the target `["", "installed just-mr"]`. That target can be built using the previously bootstrapped `just` together with the cache directory `.just`, the repository configuration `repo-conf.json`, and the build configuration `build-conf.json` left by the bootstrapping process in its build directory. This makes use of already existing cache entries and the same dependencies (typically the ones provided by the system for package builds) as for building `just`. ```sh ${BUILDDIR}/out/bin/just install \ --local-build-root ${BUILDDIR}/.just \ -C ${BUILDDIR}/repo-conf.json \ -c ${BUILDDIR}/build-conf.json \ -o ${BUILDDIR}/out/ 'installed just-mr' ``` # Installing `just-import-git`, `just-deduplicate-repos`, and `just-lock` The file `bin/just-import-git.py` is a useful Python script that allows quick generation of a multi-repository build configuration file from a simpler template for projects with dependencies provided by Git repositories. It is recommended to make this script available in your `$PATH` as `just-import-git`. Running it requires, of course, a Python3 interpreter. The file `bin/just-deduplicate-repos.py` is a useful Python script that removes duplicates from a multi-repository configuration by merging indistinguishable repositories. It is recommended to make this script available in your `$PATH` as `just-deduplicate-repos`. Running it requires, of course, a Python3 interpreter. The file `bin/just-lock.py` is a useful Python script to generate and maintain a multi-repository configuration. It is recommended to make this script available in your `$PATH` as `just-lock`. Running it requires, of course, a Python3 interpreter. just-buildsystem-justbuild-b1fb5fa/LICENSE000066400000000000000000000261361516554100600206340ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. just-buildsystem-justbuild-b1fb5fa/README.md000066400000000000000000000061601516554100600211010ustar00rootroot00000000000000# Justbuild *justbuild* is a generic build system supporting multi-repository builds. A peculiarity of the tool is the separation between global names and physical location on the one hand, and logical paths used for actions and installation on the other hand (sometimes referred to as "staging"). The language-specific information to translate high-level concepts (libraries, binaries) into individual compile actions is taken from user-defined rules described by functional expressions. Designated targets are taken entirely from cache, if the repositories transitively involved have not changed. So, by making good use of the multi-repository structure, the action graph can be kept small. Remote build execution is supported and the remote-building of cachable targets can be fully delegated to a service (provided by the tool itself); when doing so, it is not necessary to have the dependencies locally (neither as source nor as binary). ## Getting Started * The most simple way to build the `just` binary from scratch is `python3 ./bin/bootstrap.py`. For more details see the [installation guide](INSTALL.md). * Tutorial - [Getting Started](doc/tutorial/getting-started.md) - [Hello World](doc/tutorial/hello-world.md) - [Third-party dependencies](doc/tutorial/third-party-software.md) - [Tests](doc/tutorial/tests.md) - [Debugging](doc/tutorial/debugging.md) - [Targets versus `FILE`, `GLOB`, and `TREE`](doc/tutorial/target-file-glob-tree.md) - [Ensuring reproducibility](doc/tutorial/rebuild.md) - [Running linters](doc/tutorial/lint.md) - [Dependency management using Target-level Cache as a Service](doc/tutorial/just-serve.md) - [Cross compiling and testing cross-compiled targets](doc/tutorial/cross-compiling.md) - [Multi-repository configuration management](doc/tutorial/just-lock.md) * Advanced Topics - [Using protobuf](doc/tutorial/proto.md) - [How to create a single-node remote execution service](doc/tutorial/just-execute.org) - [Computed roots](doc/tutorial/computed.md) - [More build delegation through a serve endpoint](doc/tutorial/build-delegation.md) - [Invocation logging and profiling](doc/tutorial/invocation-logging.md) - [Tree overlays](doc/tutorial/tree-overlay.md) ## Documentation - [Overview](doc/concepts/overview.md) - [Build Configurations](doc/concepts/configuration.md) - [Multi-Repository Builds](doc/concepts/multi-repo.md) - [Expression Language](doc/concepts/expressions.md) - [Built-in Rules](doc/concepts/built-in-rules.md) - [User-Defined Rules](doc/concepts/rules.md) - [Documentation Strings](doc/concepts/doc-strings.md) - [Cache Pragma and Testing](doc/concepts/cache-pragma.md) - [Anonymous Targets](doc/concepts/anonymous-targets.md) - [Target-Level Caching](doc/concepts/target-cache.md) - [Target-Level Caching as a Service](doc/concepts/service-target-cache.md) - [Garbage Collection](doc/concepts/garbage.md) - [Symbolic links](doc/concepts/symlinks.md) - [Tree overlays](doc/concepts/tree-overlay.md) - [Execution properties](doc/concepts/execution-properties.md) - [Computed roots](doc/concepts/computed-roots.md) - [Profiling and Invocation Logging](doc/concepts/profiling.md) just-buildsystem-justbuild-b1fb5fa/ROOT000066400000000000000000000000001516554100600203130ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/TARGETS000066400000000000000000000174301516554100600206600ustar00rootroot00000000000000{ "": { "type": "export" , "target": "installed just" , "doc": ["The just binary."] , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "DWP" , "ARCH" , "FINAL_LDFLAGS" , "CC" , "CXX" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "SOURCE_DATE_EPOCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "VERSION_EXTRA_SUFFIX" , "PKG_CONFIG_ARGS" ] , "config_doc": { "ARCH": [ "The underlying architecture. Is taken as a default for \"HOST_ARCH\" and \"TARGET_ARCH\"." , "" , "One of \"x86\", \"x86_64\", \"arm\", \"arm64\". Defaults to \"x86_64\"." ] , "HOST_ARCH": ["The architecture on which the build actions are carried out."] , "TARGET_ARCH": ["The architecture for which to build the binary."] , "TOOLCHAIN_CONFIG": [ "The toolchain configuration." , "" , "Use field \"FAMILY\" to specify the compiler family." , "Supported values are \"unknown\" (the default), \"clang\", and \"gnu\"." , "The unknown compiler family tries to not make any assumptions on the" , "used C and C++ compilers and uses the generic \"cc\" and \"c++\" as" , "names for the respective compilers; typically used in conjunction with" , "setting \"CC\" and \"CXX\" explicitly and building for host." , "" , "Use field \"BUILD_STATIC\" to specify that binaries should be" , "statically linked. Boolean, default false." ] , "ENV": [ "Map from strings to strings. The build environment to be used for" , "build actions. Typically used to include an unusual value of PATH." ] , "FINAL_LDFLAGS": ["Compiler flags for linking the final binary."] , "DEBUG": [ "Map enabling and configuring the debug version." , "" , "The key \"USE_DEBUG_FISSION\" enables debug fission, but does not add" , "any flags. Explicitly setting it to a false value is needed to enable" , "regular debug mode." , "The key \"FISSION_CONFIG\" expects a map configuring debug fission." , " - subkey \"USE_SPLIT_DWARF\" expects a flag that, if true, adds the" , "-gsplit-dwarf compile flag." , " - subkey \"DWARF_VERSION\" expects a string that adds the" , "-gdwarf- compile flag." , " - subkey \"USE_GDB_INDEX\" expects a flag that, if true, adds the" , "-Wl,--gdb-index linker flag." , " - subkey \"USE_DEBUG_TYPES_SECTION\" expects a flag that, if true," , "adds the -fdebug-types-section compile flag." ] , "OS": [ "Operating system to build for." , "" , "Currently, the only supported value is \"linux\", which is also the" , "default." ] , "SOURCE_DATE_EPOCH": [ "If set, embed the given time stamp (in seconds since the epoch) into" , "the binary" ] , "VERSION_EXTRA_SUFFIX": [ "String to extend the version suffix with." , "" , "Should be used to indicate additional (non-upstream) changes, e.g.," , "due to packaging." ] } } , "installed just": {"type": ["@", "rules", "CC", "install-with-deps"], "targets": ["just"]} , "exported-just": { "type": "export" , "target": ["src/buildtool/main", "just"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "DEBUG" , "TOOLCHAIN_CONFIG" , "CC" , "CXX" , "ADD_CXXFLAGS" , "ADD_CFLAGS" , "AR" , "DWP" , "ENV" , "FINAL_LDFLAGS" , "SOURCE_DATE_EPOCH" , "VERSION_EXTRA_SUFFIX" , "PKG_CONFIG_ARGS" ] } , "just": { "type": "configure" , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "target": "exported-just" , "config": { "type": "let*" , "bindings": [ ["OS", {"type": "var", "name": "OS", "default": "linux"}] , ["ARCH", {"type": "var", "name": "ARCH", "default": "x86_64"}] , [ "HOST_ARCH" , { "type": "var" , "name": "HOST_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"]} } } , "installed just-mr": {"type": ["@", "rules", "CC", "install-with-deps"], "targets": ["just-mr"]} , "exported-just-mr": { "type": "export" , "target": ["src/other_tools/just_mr", "just-mr"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "DEBUG" , "TOOLCHAIN_CONFIG" , "CC" , "CXX" , "ADD_CXXFLAGS" , "ADD_CFLAGS" , "AR" , "DWP" , "ENV" , "FINAL_LDFLAGS" , "SOURCE_DATE_EPOCH" , "VERSION_EXTRA_SUFFIX" , "PKG_CONFIG_ARGS" ] } , "just-mr": { "type": "configure" , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "target": "exported-just-mr" , "config": { "type": "let*" , "bindings": [ ["OS", {"type": "var", "name": "OS", "default": "linux"}] , ["ARCH", {"type": "var", "name": "ARCH", "default": "x86_64"}] , [ "HOST_ARCH" , { "type": "var" , "name": "HOST_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"]} } } , "libgit2": { "type": "configure" , "target": ["@", "libgit2", "", "git2"] , "config": { "type": "'" , "$1": { "USE_SHA1": "OpenSSL" , "USE_SHA256": "OpenSSL" , "USE_SSH": false , "USE_HTTPS": "OpenSSL" , "USE_GSSAPI": false } } } , "libcurl": { "type": "configure" , "target": ["@", "libcurl", "", "curl"] , "config": { "type": "'" , "$1": { "ENABLE_ARES": true , "CURL_DISABLE_DICT": true , "CURL_DISABLE_FILE": true , "CURL_DISABLE_FTP": false , "CURL_DISABLE_GOPHER": true , "CURL_DISABLE_IMAP": true , "CURL_DISABLE_LDAP": true , "CURL_DISABLE_LDAPS": true , "CURL_DISABLE_MQTT": true , "CURL_DISABLE_POP3": true , "CURL_DISABLE_RTSP": true , "CURL_DISABLE_SMB": true , "CURL_DISABLE_SMTP": true , "CURL_DISABLE_TELNET": true , "CURL_DISABLE_TFTP": false , "CURL_USE_LIBPSL": false , "CURL_USE_LIBSSH2": false } } } , "libarchive": { "type": "configure" , "target": ["@", "libarchive", "", "archive"] , "config": { "type": "'" , "$1": { "ENABLE_LIBB2": false , "ENABLE_LZ4": false , "ENABLE_LZMA": true , "ENABLE_ZSTD": false , "ENABLE_LIBXML2": false , "ENABLE_EXPAT": false , "ENABLE_PCREPOSIX": false , "ENABLE_PCRE2POSIX": false , "ENABLE_LIBGCC": false , "ENABLE_CNG": false , "XATTR_PROVIDER": "gnu" , "ENABLE_ACL": false } } } , "just-ext-hdrs": { "type": "configure" , "arguments_config": ["OS", "ARCH"] , "target": ["etc/dev", "just-ext-hdrs"] , "config": { "type": "let*" , "bindings": [ ["OS", {"type": "var", "name": "OS", "default": "linux"}] , ["ARCH", {"type": "var", "name": "ARCH", "default": "x86_64"}] ] , "body": {"type": "env", "vars": ["OS", "ARCH"]} } } , "bootstrap-src": { "type": "install" , "tainted": ["test"] , "dirs": [ [["TREE", null, "rules"], "."] , [["TREE", null, "etc"], "."] , [["TREE", null, "src/buildtool"], "."] , [["TREE", null, "src/utils"], "."] ] , "deps": [ "ROOT" , "TARGETS" , "bin/bootstrap-traverser.py" , "bin/bootstrap.py" , "bin/just-mr.py" , "bin/parallel-bootstrap-traverser.py" ] } } just-buildsystem-justbuild-b1fb5fa/bin/000077500000000000000000000000001516554100600203675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/bin/bootstrap-traverser.py000077500000000000000000000152411516554100600247770ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import json import os import shutil import subprocess import sys from typing import Any, Dict, List, NoReturn, cast from argparse import ArgumentParser # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getter Json = Dict[str, Any] def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def fail(s: str) -> NoReturn: log(s) sys.exit(1) def git_hash(content: bytes) -> str: header = "blob {}\0".format(len(content)).encode('utf-8') h = hashlib.sha1() h.update(header) h.update(content) return h.hexdigest() def create_blobs(blobs: List[str], *, root: str) -> None: os.makedirs(os.path.join(root, "KNOWN")) for blob in blobs: blob_bin = blob.encode('utf-8') with open(os.path.join(root, "KNOWN", git_hash(blob_bin)), "wb") as f: f.write(blob_bin) def build_known(desc: Json, *, root: str) -> str: return os.path.join(root, "KNOWN", desc["data"]["id"]) def link(src: str, dest: str) -> None: dest = os.path.normpath(dest) os.makedirs(os.path.dirname(dest), exist_ok=True) try: os.link(src, dest) except: os.symlink(src, dest) def build_local(desc: Json, *, root: str, config: Json) -> str: repo_name = desc["data"]["repository"] repo: List[str] = config["repositories"][repo_name]["workspace_root"] rel_path = desc["data"]["path"] if repo[0] == "file": return os.path.join(repo[1], rel_path) fail("Unsupported repository root %r" % (repo, )) def build_tree(desc: Json, *, config: Json, root: str, graph: Json) -> str: tree_id = desc["data"]["id"] tree_dir = os.path.normpath(os.path.join(root, "TREE", tree_id)) if os.path.isdir(tree_dir): return tree_dir tree_dir_tmp = tree_dir + ".tmp" tree_desc = graph["trees"][tree_id] for location, desc in tree_desc.items(): link(build(desc, config=config, root=root, graph=graph), os.path.join(tree_dir_tmp, location)) # correctly handle the empty tree os.makedirs(tree_dir_tmp, exist_ok=True) shutil.copytree(tree_dir_tmp, tree_dir) return tree_dir def run_action(action_id: str, *, config: Json, root: str, graph: Json) -> str: action_dir = os.path.normpath(os.path.join(root, "ACTION", action_id)) if os.path.isdir(action_dir): return action_dir os.makedirs(action_dir) action_desc = graph["actions"][action_id] for location, desc in action_desc.get("input", {}).items(): link(build(desc, config=config, root=root, graph=graph), os.path.join(action_dir, location)) cmd = action_desc["command"] env = action_desc.get("env") log("Running %r with env %r for action %r" % (cmd, env, action_id)) for out in action_desc["output"]: os.makedirs(os.path.join(action_dir, os.path.dirname(out)), exist_ok=True) exec_dir = action_dir if "cwd" in action_desc: exec_dir = os.path.join(action_dir, action_desc["cwd"]) subprocess.run(cmd, env=env, cwd=exec_dir, check=True) return action_dir def build_action(desc: Json, *, config: Json, root: str, graph: Json) -> str: action_dir = run_action(desc["data"]["id"], config=config, root=root, graph=graph) return os.path.join(action_dir, desc["data"]["path"]) def build(desc: Json, *, config: Json, root: str, graph: Json) -> str: if desc["type"] == "TREE": return build_tree(desc, config=config, root=root, graph=graph) if desc["type"] == "ACTION": return build_action(desc, config=config, root=root, graph=graph) if desc["type"] == "KNOWN": return build_known(desc, root=root) if desc["type"] == "LOCAL": return build_local(desc, root=root, config=config) fail("Don't know how to build artifact %r" % (desc, )) def traverse(*, graph: Json, to_build: Json, out: str, root: str, config: Json) -> None: os.makedirs(out, exist_ok=True) os.makedirs(root, exist_ok=True) create_blobs(graph["blobs"], root=root) for location, artifact in to_build.items(): link(build(artifact, config=config, root=root, graph=graph), os.path.join(out, location)) def main(): parser = ArgumentParser() parser.add_argument("-C", dest="repository_config", help="Repository-description file to use", metavar="FILE") parser.add_argument("-o", dest="output_directory", help="Directory to place output to") parser.add_argument("--local-build-root", dest="local_build_root", help="Root for storing intermediate outputs", metavar="PATH") parser.add_argument("--default-workspace", dest="default_workspace", help="Workspace root to use if none is specified", metavar="PATH") (options, args) = parser.parse_known_args() if len(args) != 2: fail("usage: %r " % (sys.argv[0], )) with open(args[0]) as f: graph = json.load(f) with open(args[1]) as f: to_build = json.load(f) out = os.path.abspath(cast(str, options.output_directory or "out-boot")) root = os.path.abspath(cast(str, options.local_build_root or ".just-boot")) with open(options.repository_config or "repo-conf.json") as f: config = json.load(f) if options.default_workspace: ws_root = os.path.abspath(options.default_workspace) repos = config.get("repositories", {}).keys() for repo in repos: if not "workspace_root" in config["repositories"][repo]: config["repositories"][repo]["workspace_root"] = [ "file", ws_root ] traverse(graph=graph, to_build=to_build, out=out, root=root, config=config) if __name__ == "__main__": main() just-buildsystem-justbuild-b1fb5fa/bin/bootstrap.py000077500000000000000000000445751516554100600230000ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import json import os import shutil import subprocess import sys import tempfile import platform from pathlib import Path from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable, Dict, List, Optional, Set, cast # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters Json = Dict[str, Any] # path within the repository (constants) DEBUG = os.environ.get("DEBUG") REPOS: str = "etc/repos.in.json" MAIN_MODULE: str = "" MAIN_TARGET: str = "" MAIN_STAGE: str = "bin/just" LOCAL_LINK_DIRS_MODULE: str = "src/buildtool/main" LOCAL_LINK_DIRS_TARGET: str = "just" # architecture related configuration (global variables) g_CONF: Json = {} if 'JUST_BUILD_CONF' in os.environ: g_CONF = json.loads(os.environ['JUST_BUILD_CONF']) if "PACKAGE" in os.environ: g_CONF["ADD_CFLAGS"] = ["-Wno-error", "-Wno-pedantic"] + g_CONF.get( "ADD_CFLAGS", []) g_CONF["ADD_CXXFLAGS"] = ["-Wno-error", "-Wno-pedantic"] + g_CONF.get( "ADD_CXXFLAGS", []) ARCHS: Dict[str, str] = { 'i686': 'x86', 'x86_64': 'x86_64', 'arm': 'arm', 'aarch64': 'arm64' } if "OS" not in g_CONF: g_CONF["OS"] = platform.system().lower() if "ARCH" not in g_CONF: MACH = platform.machine() if MACH in ARCHS: g_CONF["ARCH"] = ARCHS[MACH] if 'SOURCE_DATE_EPOCH' in os.environ: g_CONF['SOURCE_DATE_EPOCH'] = int(os.environ['SOURCE_DATE_EPOCH']) g_CONFIG_PATHS: List[str] = [] if 'PKG_CONFIG_PATH' in os.environ: g_CONFIG_PATHS += [os.environ['PKG_CONFIG_PATH']] ENV: Dict[str, str] = g_CONF.setdefault("ENV", {}) if 'PKG_CONFIG_PATH' in ENV: g_CONFIG_PATHS += [ENV['PKG_CONFIG_PATH']] g_LOCALBASE: str = "/" if 'LOCALBASE' in os.environ: g_LOCALBASE = os.environ['LOCALBASE'] pkg_paths = ['lib/pkgconfig', 'share/pkgconfig'] if 'PKG_PATHS' in os.environ: pkg_paths = json.loads(os.environ['PKG_PATHS']) g_CONFIG_PATHS += [os.path.join(g_LOCALBASE, p) for p in pkg_paths] ENV['PKG_CONFIG_PATH'] = ":".join(g_CONFIG_PATHS) CONF_STRING: str = json.dumps(g_CONF) OS: str = g_CONF["OS"] ARCH: str = g_CONF["ARCH"] g_AR: str = "ar" g_CC: str = "cc" g_CXX: str = "c++" g_CFLAGS: List[str] = [] g_CXXFLAGS: List[str] = [] g_FINAL_LDFLAGS: List[str] = ["-Wl,-z,stack-size=8388608"] if "TOOLCHAIN_CONFIG" in g_CONF and "FAMILY" in g_CONF["TOOLCHAIN_CONFIG"]: if g_CONF["TOOLCHAIN_CONFIG"]["FAMILY"] == "gnu": g_CC = "gcc" g_CXX = "g++" elif g_CONF["TOOLCHAIN_CONFIG"]["FAMILY"] == "clang": g_CC = "clang" g_CXX = "clang++" if "AR" in g_CONF: g_AR = g_CONF["AR"] if "CC" in g_CONF: g_CC = g_CONF["CC"] if "CXX" in g_CONF: g_CXX = g_CONF["CXX"] if "ADD_CFLAGS" in g_CONF: g_CFLAGS = g_CONF["ADD_CFLAGS"] if "ADD_CXXFLAGS" in g_CONF: g_CXXFLAGS = g_CONF["ADD_CXXFLAGS"] if "FINAL_LDFLAGS" in g_CONF: g_FINAL_LDFLAGS += g_CONF["FINAL_LDFLAGS"] BOOTSTRAP_CC: List[str] = [g_CXX] + g_CXXFLAGS + [ "-std=c++20", "-DBOOTSTRAP_BUILD_TOOL" ] # relevant directories (global variables) g_SRCDIR: str = os.getcwd() g_WRKDIR: Optional[str] = None g_DISTDIR: List[str] = [] g_NON_LOCAL_DEPS: List[str] = [] # other global variables g_LOCAL_DEPS: bool = False def git_hash(content: bytes) -> str: header = "blob {}\0".format(len(content)).encode('utf-8') h = hashlib.sha1() h.update(header) h.update(content) return h.hexdigest() def get_checksum(filename: str) -> str: with open(filename, "rb") as f: data = f.read() return git_hash(data) def get_archive(*, distfile: str, fetch: str) -> str: # Fetch the archive, if necessary. Return path to archive for d in g_DISTDIR: candidate_path = os.path.join(d, distfile) if os.path.isfile(candidate_path): return candidate_path # Fetch to bootstrap working directory fetch_dir: str = os.path.join(cast(str, g_WRKDIR), "fetch") os.makedirs(fetch_dir, exist_ok=True) target: str = os.path.join(fetch_dir, distfile) subprocess.run(["wget", "-O", target, fetch]) return target def quote(args: List[str]) -> str: return ' '.join(["'" + arg.replace("'", "'\\''") + "'" for arg in args]) def run(cmd: List[str], *, cwd: str, **kwargs: Any) -> None: print("Running %r in %r" % (cmd, cwd), flush=True) subprocess.run(cmd, cwd=cwd, check=True, **kwargs) def setup_deps(src_wrkdir: str) -> Json: # unpack all dependencies and return a list of # additional C++ flags required with open(os.path.join(src_wrkdir, REPOS)) as f: config = json.load(f)["repositories"] include_location: str = os.path.join(cast(str, g_WRKDIR), "dep_includes") link_flags: List[str] = [] os.makedirs(include_location) for repo, total_desc in config.items(): desc: Optional[Json] = total_desc.get("repository", {}) if not isinstance(desc, dict): # Indirect definition; we will set up the repository at the # resolved place, which also has to be part of the global # repository description. continue hints = total_desc.get("bootstrap", {}) if desc.get("type") in ["archive", "zip"]: fetch = desc["fetch"] distfile = desc.get("distfile") or os.path.basename(fetch) archive = get_archive(distfile=distfile, fetch=fetch) actual_checksum = get_checksum(archive) expected_checksum = desc.get("content") if actual_checksum != expected_checksum: print("Checksum mismatch for %r. Expected %r, found %r" % (archive, expected_checksum, actual_checksum)) print("Unpacking %r from %r" % (repo, archive)) unpack_location: str = os.path.join(cast(str, g_WRKDIR), "deps", repo) os.makedirs(unpack_location) if desc["type"] == "zip": subprocess.run(["unzip", "-d", ".", archive], cwd=unpack_location, stdout=subprocess.DEVNULL) else: subprocess.run(["tar", "xf", archive], cwd=unpack_location) subdir = os.path.join(unpack_location, desc.get("subdir", ".")) include_dir = os.path.join(subdir, hints.get("include_dir", ".")) include_name = hints.get("include_name", repo) if include_name == ".": for entry in os.listdir(include_dir): os.symlink( os.path.normpath(os.path.join(include_dir, entry)), os.path.join(include_location, entry)) else: os.symlink(os.path.normpath(include_dir), os.path.join(include_location, include_name)) os_map = hints.get("os_map", dict()) arch_map = hints.get("arch_map", dict()) if "build" in hints: run([ "sh", "-c", hints["build"].format( os=os_map.get(OS, OS), arch=arch_map.get(ARCH, ARCH), cc=g_CC, cxx=g_CXX, ar=g_AR, cflags=quote(g_CFLAGS), cxxflags=quote(g_CXXFLAGS), ) ], cwd=subdir) if "link" in hints: link_flags.extend(["-L", subdir]) if "link" in hints: link_flags.extend(hints["link"]) return {"include": ["-I", include_location], "link": link_flags} def config_to_local(*, repos_file: str, link_targets_file: str) -> None: with open(repos_file) as f: repos = json.load(f) global_link_dirs: Set[str] = set() changed_file_roots: Dict[str, str] = {} backup_layers: Json = {} for repo in repos["repositories"]: if repo in g_NON_LOCAL_DEPS: continue desc = repos["repositories"][repo] repo_desc: Optional[Json] = desc.get("repository") if not isinstance(repo_desc, dict): repo_desc = {} if repo_desc.get("type") in ["archive", "zip"]: pkg_bootstrap: Json = desc.get("pkg_bootstrap", {}) desc["repository"] = { "type": "file", "path": os.path.normpath( os.path.join(g_LOCALBASE, pkg_bootstrap.get("local_path", "."))) } if "link_dirs" in pkg_bootstrap: link: List[str] = [] for entry in pkg_bootstrap["link_dirs"]: link += ["-L", os.path.join(g_LOCALBASE, entry)] global_link_dirs.add(entry) link += pkg_bootstrap.get("link", []) pkg_bootstrap["link"] = link desc["bootstrap"] = pkg_bootstrap if "pkg_bootstrap" in desc: del desc["pkg_bootstrap"] if repo_desc.get("type") == "file": pkg_bootstrap = desc.get("pkg_bootstrap", {}) if pkg_bootstrap.get("local_path") and g_NON_LOCAL_DEPS: # local layer gets changed, keep a copy backup_name: str = "ORIGINAL: " + repo backup_layers[backup_name] = { "repository": { "type": "file", "path": repo_desc.get("path") } } changed_file_roots[repo] = backup_name desc["repository"] = { "type": "file", "path": pkg_bootstrap.get("local_path", desc["repository"].get("path")) } desc["bootstrap"] = pkg_bootstrap if "pkg_bootstrap" in desc: del desc["pkg_bootstrap"] # For repos that we didn't change to local, make file roots point # to the original version, so that, in particular, the original # target root will be used. for repo in g_NON_LOCAL_DEPS: for layer in ["target_root", "rule_root", "expression_root"]: layer_ref: str = repos["repositories"][repo].get(layer) if layer_ref in changed_file_roots: repos["repositories"][repo][layer] = changed_file_roots[ layer_ref] repos["repositories"] = dict(repos["repositories"], **backup_layers) print("just-mr config rewritten to local:\n%s\n" % (json.dumps(repos, indent=2))) os.unlink(repos_file) with open(repos_file, "w") as f: json.dump(repos, f, indent=2) with open(link_targets_file) as f: target = json.load(f) main = target[LOCAL_LINK_DIRS_TARGET] link_external = [ "-L%s" % (os.path.join(g_LOCALBASE, d), ) for d in global_link_dirs ] print("External link arguments %r" % (link_external, )) main["private-ldflags"] = link_external target[LOCAL_LINK_DIRS_TARGET] = main os.unlink(link_targets_file) with open(link_targets_file, "w") as f: json.dump(target, f, indent=2) def prune_config(*, repos_file: str, empty_dir: str) -> None: with open(repos_file) as f: repos = json.load(f) for repo in repos["repositories"]: if repo in g_NON_LOCAL_DEPS: continue desc = repos["repositories"][repo] if desc.get("bootstrap", {}).get("drop"): desc["repository"] = {"type": "file", "path": empty_dir} os.unlink(repos_file) with open(repos_file, "w") as f: json.dump(repos, f, indent=2) def ignore_dst(dst: str) -> Callable[[str, List[str]], List[str]]: def ignore_(path: str, names: List[str]) -> List[str]: if os.path.normpath(path) == dst: return names for n in names: if os.path.normpath(os.path.join(path, n)) == dst: return [n] return [] return ignore_ def copy_roots(*, repos_file: str, copy_dir: str) -> None: with open(repos_file) as f: repos = json.load(f) for repo in repos["repositories"]: desc: Json = repos["repositories"][repo] to_copy = desc.get("bootstrap", {}).get("copy") if to_copy: old_root: str = desc["repository"]["path"] new_root: str = os.path.join(copy_dir, repo) for x in to_copy: src: str = os.path.join(old_root, x) dst: str = os.path.normpath(os.path.join(new_root, x)) if os.path.isdir(src): shutil.copytree(src, dst, ignore=ignore_dst(dst), symlinks=False, dirs_exist_ok=True) elif os.path.isfile(src): os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copyfile(src, dst, follow_symlinks=True) shutil.copymode(src, dst, follow_symlinks=True) desc["repository"]["path"] = new_root os.unlink(repos_file) with open(repos_file, "w") as f: json.dump(repos, f, indent=2) def bootstrap() -> None: if g_LOCAL_DEPS: print("Bootstrap build in %r from sources %r against LOCALBASE %r" % (g_WRKDIR, g_SRCDIR, g_LOCALBASE)) else: print("Bootstrapping in %r from sources %r, taking files from %r" % (g_WRKDIR, g_SRCDIR, g_DISTDIR)) os.makedirs(cast(str, g_WRKDIR), exist_ok=True) with open(os.path.join(cast(str, g_WRKDIR), "build-conf.json"), 'w') as f: json.dump(g_CONF, f, indent=2) src_wrkdir: str = os.path.normpath(os.path.join(cast(str, g_WRKDIR), "src")) shutil.copytree(g_SRCDIR, src_wrkdir, ignore=ignore_dst(src_wrkdir)) if g_LOCAL_DEPS: config_to_local(repos_file=os.path.join(src_wrkdir, REPOS), link_targets_file=os.path.join(src_wrkdir, LOCAL_LINK_DIRS_MODULE, "TARGETS")) empty_dir: str = os.path.join(cast(str, g_WRKDIR), "empty_directory") os.makedirs(empty_dir) prune_config(repos_file=os.path.join(src_wrkdir, REPOS), empty_dir=empty_dir) copy_dir: str = os.path.join(cast(str, g_WRKDIR), "copied_roots") copy_roots(repos_file=os.path.join(src_wrkdir, REPOS), copy_dir=copy_dir) dep_flags = setup_deps(src_wrkdir) # handle proto flags = ["-I", src_wrkdir] + dep_flags["include"] + [ "-I", os.path.join(g_LOCALBASE, "include") ] cpp_files: List[str] = [] for root, dirs, files in os.walk(src_wrkdir): if 'test' in dirs: dirs.remove('test') if 'execution_api' in dirs: dirs.remove('execution_api') if 'other_tools' in dirs: dirs.remove('other_tools') if 'archive' in dirs: dirs.remove('archive') if 'computed_roots' in dirs: dirs.remove('computed_roots') if 'tree_structure' in dirs: dirs.remove('tree_structure') if 'tree_operations' in dirs: dirs.remove('tree_operations') for f in files: if f.endswith(".cpp"): cpp_files.append(os.path.join(root, f)) object_files: List[str] = [] with ThreadPoolExecutor(max_workers=1 if DEBUG else None) as ts: for f in cpp_files: obj_file_name = f[:-len(".cpp")] + ".o" object_files.append(obj_file_name) cmd: List[str] = BOOTSTRAP_CC + flags + [ "-c", f, "-o", obj_file_name ] ts.submit(run, cmd, cwd=src_wrkdir) bootstrap_just: str = os.path.join(cast(str, g_WRKDIR), "bootstrap-just") final_cmd: List[str] = BOOTSTRAP_CC + g_FINAL_LDFLAGS + [ "-o", bootstrap_just ] + object_files + dep_flags["link"] run(final_cmd, cwd=src_wrkdir) CONF_FILE: str = os.path.join(cast(str, g_WRKDIR), "repo-conf.json") LOCAL_ROOT: str = os.path.join(cast(str, g_WRKDIR), ".just") os.makedirs(LOCAL_ROOT, exist_ok=True) distdirs = " --distdir=".join(g_DISTDIR) run([ "sh", "-c", "cp `./bin/just-mr.py --always-file -C %s --local-build-root=%s --distdir=%s setup just` %s" % (REPOS, LOCAL_ROOT, distdirs, CONF_FILE) ], cwd=src_wrkdir) GRAPH: str = os.path.join(cast(str, g_WRKDIR), "graph.json") TO_BUILD: str = os.path.join(cast(str, g_WRKDIR), "to_build.json") run([ bootstrap_just, "analyse", "-C", CONF_FILE, "-D", CONF_STRING, "--dump-graph", GRAPH, "--dump-artifacts-to-build", TO_BUILD, MAIN_MODULE, MAIN_TARGET ], cwd=src_wrkdir) if DEBUG: traverser = "./bin/bootstrap-traverser.py" else: traverser = "./bin/parallel-bootstrap-traverser.py" run([ traverser, "-C", CONF_FILE, "--default-workspace", src_wrkdir, GRAPH, TO_BUILD ], cwd=src_wrkdir) OUT: str = os.path.join(cast(str, g_WRKDIR), "out") run([ "./out-boot/%s" % (MAIN_STAGE, ), "install", "-C", CONF_FILE, "-D", CONF_STRING, "-o", OUT, "--local-build-root", LOCAL_ROOT, MAIN_MODULE, MAIN_TARGET ], cwd=src_wrkdir) def main(args: List[str]): global g_SRCDIR global g_WRKDIR global g_DISTDIR global g_LOCAL_DEPS global g_LOCALBASE global g_NON_LOCAL_DEPS if len(args) > 1: g_SRCDIR = os.path.abspath(args[1]) if len(args) > 2: g_WRKDIR = os.path.abspath(args[2]) if len(args) > 3: g_DISTDIR = [os.path.abspath(p) for p in args[3:]] if not g_WRKDIR: g_WRKDIR = tempfile.mkdtemp() if not g_DISTDIR: g_DISTDIR = [os.path.join(Path.home(), ".distfiles")] g_LOCAL_DEPS = "PACKAGE" in os.environ g_NON_LOCAL_DEPS = json.loads(os.environ.get("NON_LOCAL_DEPS", "[]")) bootstrap() if __name__ == "__main__": # Parse options, set g_DISTDIR main(sys.argv) just-buildsystem-justbuild-b1fb5fa/bin/format-code.sh000077500000000000000000000013361516554100600231310ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. : ${JUST_MR:=just-mr} cd $(readlink -f $(dirname $0)/..) ${JUST_MR} --main lint build -p format.diff | (patch -p1) just-buildsystem-justbuild-b1fb5fa/bin/format-json.sh000077500000000000000000000013311516554100600231630ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. : ${JUST_MR:=just-mr} cd $(readlink -f $(dirname $0)/..) ${JUST_MR} --main format-json build -p | (patch -p0) just-buildsystem-justbuild-b1fb5fa/bin/json-format.py000077500000000000000000000217421516554100600232110ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import json import difflib from functools import cmp_to_key from typing import Any, Dict, List, Tuple, Union, cast from argparse import ArgumentParser try: from pygments import console # type: ignore has_pygments = True except ImportError: has_pygments = False JSON = Union[str, int, float, bool, None, Dict[str, 'JSON'], List['JSON']] def is_simple(entry: JSON) -> bool: if isinstance(entry, list): return len(entry) == 0 if isinstance(entry, dict): return len(entry) == 0 return True def is_short(entry: JSON, indent: int) -> bool: return len(json.dumps(entry)) + indent < 80 def hdumps(entry: JSON, *, _current_indent: int = 0) -> str: if is_short(entry, _current_indent): return json.dumps(entry) if isinstance(entry, list) and entry: result = "[ " + hdumps(entry[0], _current_indent=_current_indent + 2) for x in entry[1:]: result += "\n" + " " * _current_indent + ", " result += hdumps(x, _current_indent=_current_indent + 2) result += "\n" + " " * _current_indent + "]" return result if isinstance(entry, dict) and entry: result = "{ " is_first = True for k in entry.keys(): if not is_first: result += "\n" + " " * _current_indent + ", " result += json.dumps(k) + ":" if is_simple(entry[k]): result += " " + json.dumps(entry[k]) elif is_short(entry[k], _current_indent + len(json.dumps(k)) + 4): result += " " + json.dumps(entry[k]) else: result += "\n" + " " * _current_indent + " " result += hdumps(entry[k], _current_indent=_current_indent + 2) is_first = False result += "\n" + " " * _current_indent + "}" return result return json.dumps(entry) def compare_deps(lhs: JSON, rhs: JSON) -> int: # Regular strings appear before everything else if isinstance(lhs, str) != isinstance(rhs, str): if isinstance(lhs, str): return -1 else: return 1 # Regular strings are ordered in the alphabetic order if isinstance(lhs, str) and isinstance(rhs, str): if lhs < rhs: return -1 elif lhs > rhs: return 1 if isinstance(lhs, list) and isinstance(rhs, list): # Third-party dependencies appear before other dependencies if cast(Any, lhs[0]) != cast(Any, rhs[0]): if lhs[0] == "@": return -1 if rhs[0] == "@": return 1 # Dependencies are ordered in the alphabetic order if lhs < rhs: return -1 elif lhs > rhs: return 1 return 0 def sort_list_of_dependencies(deps: JSON) -> JSON: if not isinstance(deps, list): return deps deps = sorted(deps, key=cmp_to_key(compare_deps)) # Remove duplicated dependencies i = 0 while i < len(deps) - 1: if deps[i] == deps[i + 1]: deps.pop(i + 1) else: i += 1 return deps # Get indices of intersecting entries in two sorted lists( [[lhs_index, rhs_index],...] ) # Resulting pairs are ordered in the non-descending order for both lhs and rhs. def get_intersecting_indices(lhs: JSON, rhs: JSON) -> list[Tuple[int, int]]: if not isinstance(lhs, list) or not isinstance(rhs, list): return list() intersection_indices: list[Tuple[int, int]] = list() i = 0 j = 0 while i < len(lhs) and j < len(rhs): compare_result = compare_deps(lhs[i], rhs[j]) if compare_result < 0: i += 1 elif compare_result > 0: j += 1 else: intersection_indices.append((i, j)) j += 1 return intersection_indices def sort_dependencies(content: dict[str, JSON]) -> JSON: if "deps" in content: content["deps"] = sort_list_of_dependencies(content["deps"]) if "private-deps" in content: content["private-deps"] = sort_list_of_dependencies( content["private-deps"]) # Remove intersecting dependencies between public and private dependencies, # if both are present in the target: # Typically an intersection occurs when a developer makes a private # dependency public, but forgets to remove the private-deps entry. # That's why private-deps entries are deleted here. if "deps" in content and "private-deps" in content: intersection = get_intersecting_indices(content["deps"], content["private-deps"]) for _, j in reversed(intersection): cast(list[JSON], content["private-deps"]).pop(j) return content def sort_targets_dependencies(data: str) -> str: targets: dict[str, JSON] = json.loads(data) for target_name, content in targets.items(): targets[target_name] = sort_dependencies(cast(Dict[str, JSON], content)) return json.dumps(targets) def color_diff(before: str, after: str): next_lines = 0 lines: List[str] = [] for line in difflib.ndiff(before.splitlines(keepends=True), after.splitlines(keepends=True)): if line.startswith('+'): next_lines = 3 prev_lines = lines[-3:] lines.clear() yield "".join(prev_lines + [ console.colorize("green", line) # type: ignore if has_pygments else line ]) elif line.startswith('-'): next_lines = 3 prev_lines = lines[-3:] lines.clear() yield "".join(prev_lines + [ console.colorize("red", line) # type: ignore if has_pygments else line ]) elif line.startswith('?'): next_lines = 3 prev_lines = lines[-3:] lines.clear() yield "".join(prev_lines + [ console.colorize("blue", line) # type: ignore if has_pygments else line ]) else: if next_lines > 0: next_lines -= 1 yield line else: lines.append(line) if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("in_file", help="input file (omit for stdin)", nargs='?', default=None) parser.add_argument("-c", "--check", action='store_true', help="just verify format of input", default=False) parser.add_argument("-d", "--diff", action='store_true', help="with -c, print colored diff", default=False) parser.add_argument("-i", "--in-place", action='store_true', help="modify input file in-place", default=False) parser.add_argument("-s", "--sort", action='store_true', help="sort dependencies and remove duplicates", default=False) options = parser.parse_args() if options.in_file: with open(options.in_file, 'r') as f: data = f.read() else: data = sys.stdin.read() try: data_formatted: str = data if options.sort: data_formatted = sort_targets_dependencies(data_formatted) data_formatted: str = hdumps(json.loads(data_formatted)) except Exception as e: # if the file contains syntax errors, we print the file name if options.in_file: print("Found syntax issues in", options.in_file) print(e) exit(1) data_newline = data_formatted + '\n' if options.check: if data != data_newline: print("Found format issues" + ( (" in file: " + options.in_file) if options.in_file else ":")) if options.diff: print("".join(color_diff(data, data_newline))) exit(1) exit(0) if options.in_place and options.in_file: out_file = f"{options.in_file}.out" with open(out_file, 'w') as f: f.write(data_newline) os.rename(out_file, options.in_file) exit(0) print(data_formatted) just-buildsystem-justbuild-b1fb5fa/bin/just-deduplicate-repos.py000077500000000000000000000265111516554100600253450ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import sys from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union, cast # generic JSON type Json = Any def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def fail(s: str, exit_code: int = 1) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) def bisimilar_repos(repos: Json) -> List[List[str]]: """Compute the maximal bisimulation between the repositories and return the bisimilarity classes.""" bisim: Dict[Tuple[str, str], Json] = {} def is_different(name_a: str, name_b: str) -> bool: pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) return bisim.get(pos, {}).get("different", False) def mark_as_different(name_a: str, name_b: str): nonlocal bisim pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) entry = bisim.get(pos, {}) if entry.get("different"): return bisim[pos] = dict(entry, **{"different": True}) also_different: List[Tuple[str, str]] = entry.get("different_if", []) for a, b in also_different: mark_as_different(a, b) def register_dependency(name_a: str, name_b: str, dep_a: str, dep_b: str): pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) entry = bisim.get(pos, {}) deps: List[Tuple[str, str]] = entry.get("different_if", []) deps.append((dep_a, dep_b)) bisim[pos] = dict(entry, **{"different_if": deps}) def roots_equal(a: Json, b: Json, name_a: str, name_b: str) -> bool: if a["type"] != b["type"]: return False if a["type"] == "file": return a["path"] == b["path"] elif a["type"] in ["archive", "zip"]: return (a["content"] == b["content"] and a.get("subdir", ".") == b.get("subdir", ".")) elif a["type"] == "git": return (a["commit"] == b["commit"] and a.get("subdir", ".") == b.get("subdir", ".")) elif a["type"] in ["computed", "tree structure"]: if a["type"] == "computed": if (a.get("config", {}) != b.get("config", {}) or a["target"] != b["target"]): return False if a["repo"] == b["repo"]: return True elif is_different(a["repo"], b["repo"]): return False else: # equality pending target repo equality register_dependency(a["repo"], b["repo"], name_a, name_b) return True else: # unknown repository type, the only safe way is to test # for full equality return a == b def get_root(repos: Json, name: str, *, root_name: str = "repository", default_root: Optional[Json] = None) -> Json: root = repos[name].get(root_name) if root is None: if default_root is not None: return default_root else: fail("Did not find mandatory root %s" % (name, )) if isinstance(root, str): return get_root(repos, root) return root def repo_roots_equal(repos: Json, name_a: str, name_b: str) -> bool: if name_a == name_b: return True root_a = None root_b = None for root_name in [ "repository", "target_root", "rule_root", "expression_root" ]: root_a = get_root(repos, name_a, root_name=root_name, default_root=root_a) root_b = get_root(repos, name_b, root_name=root_name, default_root=root_b) if not roots_equal(root_a, root_b, name_a, name_b): return False for file_name, default_name in [("target_file_name", "TARGETS"), ("rule_file_name", "RULES"), ("expression_file_name", "EXPRESSIONS") ]: fname_a = repos[name_a].get(file_name, default_name) fname_b = repos[name_b].get(file_name, default_name) if fname_a != fname_b: return False return True names = sorted(repos.keys()) for j in range(len(names)): b = names[j] for i in range(j): a = names[i] if is_different(a, b): continue if not repo_roots_equal(repos, a, b): mark_as_different(a, b) continue links_a = repos[a].get("bindings", {}) links_b = repos[b].get("bindings", {}) if set(links_a.keys()) != set(links_b.keys()): mark_as_different(a, b) continue for link in links_a.keys(): next_a = links_a[link] next_b = links_b[link] if next_a != next_b: if is_different(next_a, next_b): mark_as_different(a, b) continue else: # equality pending binding equality register_dependency(next_a, next_b, a, b) classes: List[List[str]] = [] done: Dict[str, bool] = {} for j in reversed(range(len(names))): name_j = names[j] if done.get(name_j): continue c = [name_j] for i in range(j): name_i = names[i] if not bisim.get((name_i, name_j), {}).get("different"): c.append(name_i) done[name_i] = True classes.append(c) return classes def dedup(repos: Json, user_keep: List[str]) -> Json: keep = set(user_keep) main = repos.get("main") if isinstance(main, str): keep.add(main) def choose_representative(c: List[str]) -> str: """Out of a bisimilarity class chose a main representative""" candidates = c # Keep a repository with a proper root, if any of those has a root. # In this way, we're not losing actual roots. with_root = [ n for n in candidates if isinstance(repos["repositories"][n]["repository"], dict) ] if with_root: candidates = with_root # Prefer to choose a repository we have to keep anyway keep_entries = set(candidates) & keep if keep_entries: candidates = list(keep_entries) return sorted(candidates, key=lambda s: (s.count("/"), len(s), s))[0] def merge_pragma(rep: str, merged: List[str]) -> Json: desc = cast(Union[str, Dict[str, Json]], repos["repositories"][rep]["repository"]) if not isinstance(desc, dict): return desc pragma = desc.get("pragma", {}) # Clear pragma absent unless all merged repos that are not references # have the pragma absent: bool = pragma.get("absent", False) for c in merged: alt_desc = cast(Union[str, Dict[str, Json]], repos["repositories"][c]["repository"]) if (isinstance(alt_desc, dict)): absent = \ absent and alt_desc.get("pragma", {}).get("absent", False) pragma = dict(pragma, **{"absent": absent}) if not absent: del pragma["absent"] # Add pragma to_git if at least one of the merged repos requires it to_git = pragma.get("to_git", False) for c in merged: alt_desc = cast(Union[str, Dict[str, Json]], repos["repositories"][c]["repository"]) if (isinstance(alt_desc, dict)): to_git = \ to_git or alt_desc.get("pragma", {}).get("to_git", False) pragma = dict(pragma, **{"to_git": to_git}) if not to_git: del pragma["to_git"] # Update the pragma desc = dict(desc, **{"pragma": pragma}) if not pragma: del desc["pragma"] return desc bisim = bisimilar_repos(repos["repositories"]) renaming: Dict[str, str] = {} updated_repos: Json = {} for c in bisim: if len(c) == 1: continue rep = choose_representative(c) updated_repos[rep] = merge_pragma(rep, c) for repo in c: if ((repo not in keep) and (repo != rep)): renaming[repo] = rep def final_root_reference(name: str) -> str: """For a given repository name, return a name than can be used to name root in the final repository configuration.""" root: Json = repos["repositories"][name]["repository"] if isinstance(root, dict): # actual root; can still be merged into a different once, but only # one with a proper root as well. return renaming.get(name, name) if isinstance(root, str): return final_root_reference(root) fail("Invalid root found for %r: %r" % (name, root)) new_repos: Json = {} for name in repos["repositories"].keys(): if name not in renaming: desc = repos["repositories"][name] if name in updated_repos: desc = dict(desc, **{"repository": updated_repos[name]}) if "bindings" in desc: bindings = desc["bindings"] new_bindings = {} for k, v in bindings.items(): if v in renaming: new_bindings[k] = renaming[v] else: new_bindings[k] = v desc = dict(desc, **{"bindings": new_bindings}) new_roots: Json = {} for root in ["repository", "target_root", "rule_root"]: root_val: Json = desc.get(root) if isinstance(root_val, str) and (root_val in renaming): new_roots[root] = final_root_reference(root_val) desc = dict(desc, **new_roots) # Update target repos of precomputed roots: if isinstance(desc.get("repository"), dict): repo_root: Json = desc.get("repository") if repo_root["type"] in ["computed", "tree structure"] and \ repo_root["repo"] in renaming: repo_root = \ dict(repo_root, **{"repo": renaming[repo_root["repo"]]}) desc = dict(desc, **{"repository": repo_root}) new_repos[name] = desc return dict(repos, **{"repositories": new_repos}) if __name__ == "__main__": orig = json.load(sys.stdin) final = dedup(orig, sys.argv[1:]) print(json.dumps(final)) just-buildsystem-justbuild-b1fb5fa/bin/just-import-git.py000077500000000000000000000367611516554100600240370ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import shutil import sys import tempfile from argparse import ArgumentParser, Namespace from pathlib import Path from typing import Any, Dict, List, NoReturn, Optional, Set, Tuple, cast # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters Json = Dict[str, Any] def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def fail(s: str, exit_code: int = 1) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) MARKERS: List[str] = [".git", "ROOT", "WORKSPACE"] SYSTEM_ROOT: str = os.path.abspath(os.sep) ALT_DIRS: List[str] = ["target_root", "rule_root", "expression_root"] DEFAULT_CONFIG_LOCATIONS: List[Dict[str, str]] = [{ "root": "workspace", "path": "repos.json" }, { "root": "workspace", "path": "etc/repos.json" }, { "root": "home", "path": ".just-repos.json" }, { "root": "system", "path": "etc/just-repos.json" }] def run_cmd(cmd: List[str], *, env: Optional[Any] = None, stdout: Optional[Any] = subprocess.DEVNULL, stdin: Optional[Any] = None, cwd: str): result = subprocess.run(cmd, cwd=cwd, env=env, stdout=stdout, stdin=stdin) if result.returncode != 0: fail("Command %s in %s failed" % (cmd, cwd)) return result.stdout def find_workspace_root(path: Optional[str] = None) -> Optional[str]: def is_workspace_root(path: str) -> bool: for m in MARKERS: if os.path.exists(os.path.join(path, m)): return True return False if not path: path = os.getcwd() while True: if is_workspace_root(path): return path if path == SYSTEM_ROOT: return None path = os.path.dirname(path) def read_location(location: Dict[str, str], root: Optional[str] = None) -> str: search_root = location.get("root", None) search_path = location.get("path", None) fs_root = None if search_root == "workspace": if root: fs_root = root else: fs_root = find_workspace_root() if not root: if search_root == "home": fs_root = Path.home() if search_root == "system": fs_root = SYSTEM_ROOT if fs_root: return os.path.realpath( os.path.join(cast(str, fs_root), cast(str, search_path))) return "/" # certainly not a file def get_repository_config_file(root: Optional[str] = None) -> Optional[str]: for location in DEFAULT_CONFIG_LOCATIONS: path = read_location(location, root=root) if path and os.path.isfile(path): return path def get_base_config(repository_config: Optional[str]) -> Json: if repository_config == "-": return json.load(sys.stdin) if not repository_config: repository_config = get_repository_config_file() if (repository_config): with open(repository_config) as f: return json.load(f) fail('Could not get base config') def clone( url: str, branch: str, *, mirrors: List[str], inherit_env: List[str], ) -> Tuple[str, Dict[str, Any], str]: # clone the given git repository, checkout the specified # branch, and return the checkout location workdir: str = tempfile.mkdtemp() run_cmd(["git", "clone", "-b", branch, "--depth", "1", url, "src"], cwd=workdir) srcdir: str = os.path.join(workdir, "src") commit: str = run_cmd(["git", "log", "-n", "1", "--pretty=%H"], cwd=srcdir, stdout=subprocess.PIPE).decode('utf-8').strip() log("Importing commit %s" % (commit, )) repo: Dict[str, Any] = { "type": "git", "repository": url, "branch": branch, "commit": commit, } if mirrors: repo = dict(repo, **{"mirrors": mirrors}) if inherit_env: repo = dict(repo, **{"inherit env": inherit_env}) return srcdir, repo, workdir def get_repo_to_import(config: Json) -> str: """From a given repository config, take the main repository.""" if config.get("main") is not None: return cast(str, config.get("main")) repos = config.get("repositories", {}).keys() if repos: return cast(str, sorted(repos)[0]) fail("Config does not contain any repositories; unsure what to import") def get_target_if_computed_repo(repo: Any, repos_config: Json) -> Optional[str]: """If repository is computed, return the target repository name.""" while isinstance(repo, str): repo = repos_config[repo]["repository"] if repo.get("type") in ["computed", "tree structure"]: return cast(str, repo.get("repo")) return None def repos_to_import(repos_config: Json, entry: str, known: Set[str]) -> Tuple[List[str], List[str]]: """Compute the set of transitively reachable repositories and the collection of repositories additionally needed as they serve as layers for the repositories to import.""" to_import: Set[str] = set() extra_imports: Set[str] = set() def visit(name: str) -> None: # skip any existing or already visited repositories if name in known or name in to_import: return repo_desc: Json = repos_config.get(name, {}) # if proper import, visit bindings, which are fully imported if name not in extra_imports: to_import.add(name) vals = cast(Dict[str, str], repo_desc.get("bindings", {})).values() for n in vals: extra_imports.discard(n) visit(n) repo = repo_desc.get("repository") if isinstance(repo, str): # visit referred repository, but skip bindings if repo not in known and repo not in to_import: extra_imports.add(repo) visit(repo) else: # if computed, visit the referred repository target = get_target_if_computed_repo(repo, repos_config) if target is not None: extra_imports.discard(target) visit(target) # add layers as extra imports, but referred repositories of computed # layers need to be fully imported for layer in ALT_DIRS: if layer in repo_desc: extra: str = repo_desc[layer] if extra not in known and extra not in to_import: extra_imports.add(extra) extra_target = get_target_if_computed_repo( repos_config.get(extra, {}).get("repository", {}), repos_config) if extra_target is not None: extra_imports.discard(extra_target) visit(extra_target) visit(entry) return list(to_import), list(extra_imports) def name_imports(to_import: List[str], extra_imports: List[str], existing: Set[str], base_name: str, main: Optional[str] = None) -> Dict[str, str]: """Assign names to the repositories to import in such a way that no conflicts arise.""" assign: Dict[str, str] = {} def find_name(name: str) -> str: base: str = "%s/%s" % (base_name, name) if (base not in existing) and (base not in assign): return base count: int = 0 while True: count += 1 candidate: str = base + " (%d)" % count if (candidate not in existing) and (candidate not in assign): return candidate if main is not None and (base_name not in existing): assign[main] = base_name to_import = [x for x in to_import if x != main] extra_imports = [x for x in extra_imports if x != main] for repo in to_import + extra_imports: assign[repo] = find_name(repo) return assign def rewrite_repo(repo_spec: Json, *, remote: Dict[str, Any], assign: Json, absent: bool, as_layer: bool = False) -> Json: new_spec: Json = {} repo = repo_spec.get("repository", {}) if isinstance(repo, str): repo = assign[repo] elif repo.get("type") == "file": changes = {} # take subdir subdir: str = os.path.normpath(repo.get("path", ".")) if subdir != ".": changes["subdir"] = subdir # keep ignore special and absent pragmas pragma = {} if cast(Json, repo).get("pragma", {}).get("special", None) == "ignore": pragma["special"] = "ignore" if cast(Json, repo).get("pragma", {}).get("absent", False): pragma["absent"] = True if pragma: changes["pragma"] = pragma repo = dict(remote, **changes) elif repo.get("type") == "distdir": existing_repos: List[str] = repo.get("repositories", []) new_repos = [assign[k] for k in existing_repos] repo = dict(repo, **{"repositories": new_repos}) elif repo.get("type") in ["computed", "tree structure"]: target: str = repo.get("repo", None) repo = dict(repo, **{"repo": assign[target]}) if absent and isinstance(repo, dict): repo["pragma"] = dict(repo.get("pragma", {}), **{"absent": True}) new_spec["repository"] = repo # rewrite other roots and bindings, if actually needed to be imported if not as_layer: for key in ["target_root", "rule_root", "expression_root"]: if key in repo_spec: new_spec[key] = assign[repo_spec[key]] for key in [ "target_file_name", "rule_file_name", "expression_file_name" ]: if key in repo_spec: new_spec[key] = repo_spec[key] bindings = repo_spec.get("bindings", {}) new_bindings = {} for k, v in bindings.items(): new_bindings[k] = assign[v] if new_bindings: new_spec["bindings"] = new_bindings return new_spec def handle_import(args: Namespace) -> Json: base_config: Json = get_base_config(args.repository_config) base_repos: Json = base_config.get("repositories", {}) srcdir, remote, to_cleanup = clone( args.URL, args.branch, mirrors=args.mirrors, inherit_env=args.inherit_env, ) if args.foreign_repository_config: foreign_config_file = os.path.join(srcdir, args.foreign_repository_config) else: foreign_config_file = get_repository_config_file(srcdir) foreign_config: Json = {} if args.plain: foreign_config = { "main": "", "repositories": { "": { "repository": { "type": "file", "path": "." } } } } else: if (foreign_config_file): with open(foreign_config_file) as f: foreign_config = json.load(f) else: fail('Could not get repository config file') foreign_repos: Json = foreign_config.get("repositories", {}) if args.foreign_repository_name: foreign_name = cast(str, args.foreign_repository_name) else: foreign_name = get_repo_to_import(foreign_config) import_map: Json = {} for theirs, ours in args.import_map: import_map[theirs] = ours main_repos, extra_imports = repos_to_import(foreign_repos, foreign_name, set(import_map.keys())) extra_repos = sorted([x for x in main_repos if x != foreign_name]) ordered_imports: List[str] = [foreign_name] + extra_repos extra_imports = sorted(extra_imports) import_name = foreign_name if args.import_as is not None: import_name = args.import_as assign: Dict[str, str] = name_imports( ordered_imports, extra_imports, set(base_repos.keys()), import_name, main=foreign_name, ) log("Importing %r as %r" % (foreign_name, import_name)) log("Transitive dependencies to import: %r" % (extra_repos, )) log("Repositories imported as layers: %r" % (extra_imports, )) total_assign = dict(assign, **import_map) for repo in ordered_imports: base_repos[assign[repo]] = rewrite_repo( foreign_repos[repo], remote=remote, assign=total_assign, absent=args.absent, ) for repo in extra_imports: base_repos[assign[repo]] = rewrite_repo( foreign_repos[repo], remote=remote, assign=total_assign, absent=args.absent, as_layer=True, ) base_config["repositories"] = base_repos shutil.rmtree(to_cleanup) return base_config def main(): parser = ArgumentParser( prog="just-import-deps", description="Import a dependency transitively into a given" + " multi-repository configuration") parser.add_argument("-C", dest="repository_config", help="Repository-description file to import into", metavar="FILE") parser.add_argument( "-b", dest="branch", help="The branch of the remote repository to import and follow", metavar="branch", default="master") parser.add_argument( "-R", dest="foreign_repository_config", help="Repository-description file in the repository to import", metavar="relative-path") parser.add_argument( "--plain", action="store_true", help="Pretend the remote repository description is the canonical" + " single-repository one", ) parser.add_argument( "--absent", action="store_true", help="Import repository and all its dependencies as absent.") parser.add_argument( "--as", dest="import_as", help="Name prefix to import the foreign repository as", metavar="NAME", ) parser.add_argument( "--map", nargs=2, dest="import_map", help= "Map the specified foreign repository to the specified existing repository", action="append", default=[]) parser.add_argument("--mirror", dest="mirrors", help="Alternative fetch locations for the repository", action="append", default=[], metavar="URL") parser.add_argument( "--inherit-env", dest="inherit_env", help="Environment variables to inherit when calling git to fetch", action="append", default=[], metavar="VAR") parser.add_argument('URL') parser.add_argument('foreign_repository_name', nargs='?') args = parser.parse_args() new_config = handle_import(args) print(json.dumps(new_config)) if __name__ == "__main__": main() just-buildsystem-justbuild-b1fb5fa/bin/just-lock.py000077500000000000000000003640731516554100600226740ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import fcntl import hashlib import json import multiprocessing import os import shutil import stat import subprocess import sys import tempfile import time import zlib from argparse import ArgumentParser, ArgumentError, RawTextHelpFormatter from pathlib import Path from typing import Any, Dict, List, NoReturn, Optional, Set, TextIO, Tuple, Union, cast from enum import Enum from concurrent.futures import ThreadPoolExecutor # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters Json = Dict[str, Any] ### # Constants ## MARKERS: List[str] = [".git", "ROOT", "WORKSPACE"] SYSTEM_ROOT: str = os.path.abspath(os.sep) ALT_DIRS: List[str] = ["target_root", "rule_root", "expression_root"] REPO_ROOTS: List[str] = ["repository"] + ALT_DIRS REPO_KEYS_TO_KEEP: List[str] = [ "target_file_name", "rule_file_name", "expression_file_name", "bindings" ] + ALT_DIRS DEFAULT_BUILD_ROOT: str = os.path.join(Path.home(), ".cache/just") DEFAULT_GIT_BIN: str = "git" # to be taken from PATH DEFAULT_LAUNCHER: List[str] = ["env", "--"] DEFAULT_REPO: Json = {"": {"repository": {"type": "file", "path": "."}}} DEFAULT_INPUT_CONFIG_NAME: str = "repos.in.json" DEFAULT_JUSTMR_CONFIG_NAME: str = "repos.json" DEFAULT_CONFIG_DIRS: List[str] = [".", "./etc"] """Directories where to look for configuration file inside a root""" DEFAULT_JUST: str = "just" GIT_NOBODY_ENV: Dict[str, str] = { "GIT_AUTHOR_DATE": "1970-01-01T00:00Z", "GIT_AUTHOR_NAME": "Nobody", "GIT_AUTHOR_EMAIL": "nobody@example.org", "GIT_COMMITTER_DATE": "1970-01-01T00:00Z", "GIT_COMMITTER_NAME": "Nobody", "GIT_COMMITTER_EMAIL": "nobody@example.org", "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", } class ObjectType(Enum): FILE = 1 EXEC = 2 LINK = 3 DIR = 4 SUPPORTED_BLOB_TYPES: List[ObjectType] = [ ObjectType.FILE, ObjectType.EXEC, ObjectType.LINK ] SHA1_SIZE_BYTES: int = 20 class LogLimit(Enum): ERROR = 1 WARN = 2 INFO = 3 LOGGER_MAP: Dict[LogLimit, Tuple[str, str]] = { # Color is fmt::color::red LogLimit.ERROR: (f"\033[38;2;255;0;0mERROR:\033[0m", 6 * " "), # Color is fmt::color::orange LogLimit.WARN: (f"\033[38;2;255;0;0mWARN:\033[0m", 5 * " "), # Color is fmt::color::lime_green LogLimit.INFO: (f"\033[38;2;50;205;50mINFO:\033[0m", 5 * " ") } """Mapping from log limit to pair of colored prefix and continuation prefix.""" ### # Global vars ## g_ROOT: str = DEFAULT_BUILD_ROOT """The configured local build root""" g_JUST: str = DEFAULT_JUST """The path to the 'just' binary""" g_GIT: str = DEFAULT_GIT_BIN """Git binary to use""" g_LAUNCHER: List[str] = DEFAULT_LAUNCHER """Local launcher to use for commands provided in imports""" g_CLONE_MAP: Dict[str, Tuple[str, List[str]]] = {} """Mapping from local path to pair of repository name and bindings chain for cloning""" ### # System utils ## def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def formatted_log(log_limit: LogLimit, msg: str) -> None: parts: List[str] = msg.rstrip('\n').split('\n') new_msg: str = "%s %s" % (LOGGER_MAP[log_limit][0], parts[0]) for part in parts[1:]: new_msg += "\n%s %s" % (LOGGER_MAP[log_limit][1], part) log(new_msg) def fail(s: str, exit_code: int = 1) -> NoReturn: """Log as error and exit. Matches the color scheme of 'just-mr'.""" formatted_log(LogLimit.ERROR, s) sys.exit(exit_code) def warn(s: str) -> None: """Log as warning. Matches the color scheme of 'just-mr'.""" formatted_log(LogLimit.WARN, s) def report(s: Optional[str]) -> None: """Log as information message. Matches the color scheme of 'just-mr'.""" if s is None: log("") else: formatted_log(LogLimit.INFO, s) def run_cmd( cmd: List[str], *, env: Optional[Any] = None, stdout: Optional[Any] = subprocess.DEVNULL, # ignore output by default stderr: Optional[Any] = subprocess.PIPE, # capture errors by default stdin: Optional[Any] = None, input: Optional[bytes] = None, cwd: str, attempts: int = 1, fail_context: Optional[str] = None, ) -> Tuple[bytes, int]: """Run a specific command. If fail_context string given, exit on failure. Expects fail_context to end in a newline.""" attempts = max(attempts, 1) # at least one attempt result: Any = None for _ in range(attempts): result = subprocess.run(cmd, cwd=cwd, env=env, stdout=stdout, stderr=stderr, stdin=stdin, input=input) if result.returncode == 0: return result.stdout, result.returncode # return successful result if fail_context is not None: fail("%sCommand %s in %s failed after %d attempt%s with:\n%s" % (fail_context, cmd, cwd, attempts, "" if attempts == 1 else "s", result.stderr)) return result.stderr, result.returncode # return result of last failure def try_rmtree(tree: str) -> None: """Safely remove a directory tree.""" for _ in range(10): try: shutil.rmtree(tree) return except: time.sleep(1.0) fail("Failed to remove %s" % (tree, )) def lock_acquire(fpath: str, is_shared: bool = False) -> TextIO: """Acquire a lock on a file descriptor. It opens a stream with shared access for given file path and returns it to keep it alive. The lock can only be released by calling lock_release() on this stream.""" if os.path.exists(fpath): if os.path.isdir(fpath): fail("Lock path %s is a directory!" % (fpath, )) else: os.makedirs(Path(fpath).parent, exist_ok=True) lockfile = open(fpath, "a+") # allow shared read and create if on first try fcntl.flock(lockfile.fileno(), fcntl.LOCK_SH if is_shared else fcntl.LOCK_EX) return lockfile def lock_release(lockfile: TextIO) -> None: """Release lock on the file descriptor of the given open stream, then close the stream. Expects the argument to be the output of a previous lock_acquire() call.""" fcntl.flock(lockfile.fileno(), fcntl.LOCK_UN) lockfile.close() ### # Storage utils ## def create_tmp_dir(*, type: str) -> str: """Create unique temporary directory inside the local build root and return its path. Caller is responsible for deleting it and its content once not needed anymore.""" root = os.path.join(g_ROOT, "tmp-workspaces", type) os.makedirs(root, exist_ok=True) return tempfile.mkdtemp(dir=root) ### # Config utils ## def find_workspace_root() -> Optional[str]: """Find the workspace root of the current working directory.""" def is_workspace_root(path: str) -> bool: for m in MARKERS: if os.path.exists(os.path.join(path, m)): return True return False path: str = os.getcwd() while True: if is_workspace_root(path): return path if path == SYSTEM_ROOT: return None path = os.path.dirname(path) def get_repository_config_file(filename: str, root: Optional[str] = None) -> Optional[str]: """Get a named configuration file relative to a root.""" if not root: root = find_workspace_root() if not root: return None for dir in DEFAULT_CONFIG_DIRS: path: str = os.path.realpath(os.path.join(root, dir, filename)) if os.path.isfile(path): return path ### # Git utils ## def gc_repo_lock_acquire(is_shared: bool = False) -> TextIO: """Acquire garbage collector file lock for the Git cache.""" # use same naming scheme as in Just return lock_acquire(os.path.join(g_ROOT, "repositories/gc.lock"), is_shared) def git_root(*, upstream: Optional[str]) -> str: """Get the root of specified upstream repository. Passing None always returns the root of the Git cache repository. No checks are made on the returned path.""" return (os.path.join(g_ROOT, "repositories/generation-0/git") if upstream is None else upstream) def git_keep(commit: str, *, upstream: Optional[str], fail_context: str) -> None: """Keep commit by tagging it. It is a user error if the referenced Git repository does not exist.""" root: str = git_root(upstream=upstream) # acquire exclusive lock lockfile = lock_acquire(os.path.join(Path(root).parent, "init_open.lock")) # tag commit git_env = {**os.environ.copy(), **GIT_NOBODY_ENV} run_cmd(g_LAUNCHER + [ g_GIT, "tag", "-f", "-m", "Keep referenced tree alive", "keep-%s" % (commit, ), commit ], cwd=root, env=git_env, attempts=3, fail_context=fail_context) # release exclusive lock lock_release(lockfile) def ensure_git_init(*, upstream: Optional[str], init_bare: bool = True, fail_context: str) -> None: """Ensure Git repository given by upstream is initialized. Use an exclusive lock to ensure the initialization happens only once.""" root: str = git_root(upstream=upstream) # acquire exclusive lock; use same naming scheme as in Just lockfile = lock_acquire(os.path.join(Path(root).parent, "init_open.lock")) # do the critical work if os.path.exists(root): return os.makedirs(root) git_init_cmd: List[str] = [g_GIT, "init"] if init_bare: git_init_cmd += ["--bare"] run_cmd(g_LAUNCHER + git_init_cmd, cwd=root, fail_context=fail_context) # release the exclusive lock lock_release(lockfile) def git_commit_present(commit: str, *, upstream: Optional[str]) -> bool: """Check if commit is present in specified Git repository. Does not require the repository to exist, in which case it returns false.""" root: str = git_root(upstream=upstream) return (os.path.exists(root) and run_cmd(g_LAUNCHER + [g_GIT, "show", "--oneline", commit], stdout=subprocess.DEVNULL, cwd=root, fail_context=None)[1] == 0) def git_url_is_path(url: str) -> Optional[str]: """Get the path a URL refers to if it is in a supported path format, and None otherwise.""" if url.startswith('/'): return url if url.startswith('./'): return url[len('./'):] if url.startswith('file://'): return url[len('file://'):] return None def git_fetch(*, from_repo: Optional[str], to_repo: Optional[str], fetchable: str, fail_context: Optional[str]) -> bool: """Fetch from a given repository a fetchable object (branch or commit) into another repository. A None value for a repository means the Git cache repository is used. Returns success flag of fetch command. It is a user error if the referenced Git repositories do not exist.""" if from_repo is None: from_repo = git_root(upstream=None) else: path_url = git_url_is_path(from_repo) if path_url is not None: from_repo = os.path.abspath(path_url) return run_cmd(g_LAUNCHER + [ g_GIT, "fetch", "--no-auto-gc", "--no-write-fetch-head", from_repo, fetchable ], cwd=git_root(upstream=to_repo), fail_context=fail_context)[1] == 0 def type_to_perm(obj_type: ObjectType) -> str: """Mapping from Git object type to filesystem permission string.""" if obj_type == ObjectType.DIR: return "40000" elif obj_type == ObjectType.LINK: return "120000" elif obj_type == ObjectType.EXEC: return "100755" elif obj_type == ObjectType.FILE: return "100644" fail("Unexpected object type %r" % (obj_type, )) def type_to_string(obj_type: ObjectType) -> str: """Mapping from Git object type to human-readable string.""" if obj_type == ObjectType.DIR: return "DIR" elif obj_type == ObjectType.LINK: return "LINK" elif obj_type == ObjectType.EXEC: return "EXEC" elif obj_type == ObjectType.FILE: return "FILE" fail("Unexpected object type %r" % (obj_type, )) def write_data_to_repo(repo_root: str, data: bytes, *, as_type: str) -> bytes: """Write content of an object of certain type into given repository. Returns the raw id of the written object.""" # Get hash and header to be stored h, header = git_hash(data, type=as_type) # Write repository object obj_dir = "{}/.git/objects/{}".format(repo_root, h[0:2]) obj_file = "{}/{}".format(obj_dir, h[2:]) os.makedirs(obj_dir, exist_ok=True) with open(obj_file, "wb") as f: f.write(zlib.compress(header + data)) return bytes.fromhex(h) # raw id def write_blob_to_repo(repo_root: str, data: bytes) -> bytes: """Write blob into given Git repository.""" return write_data_to_repo(repo_root, data, as_type="blob") def write_tree_to_repo(repo_root: str, entries: Dict[str, Tuple[bytes, ObjectType]]) -> bytes: """Write tree entries into given Git repository. Tree entries have as key their filename and as value a tuple of raw id and object type. They must be sorted by filename.""" tree_content: bytes = b"" for fname, entry in sorted(entries.items()): if entry[1] == ObjectType.DIR: # remove any trailing '/' if fname[-1] == '/': fname = fname[:-1] tree_content += "{} {}\0".format(type_to_perm(entry[1]), fname).encode('utf-8') + entry[0] return write_data_to_repo(repo_root, tree_content, as_type="tree") def path_to_type(fpath: str) -> ObjectType: """Get type of given filesystem entry.""" if os.path.islink(fpath): return ObjectType.LINK elif os.path.isdir(fpath): return ObjectType.DIR elif os.path.isfile(fpath): if os.access(fpath, os.X_OK): return ObjectType.EXEC else: return ObjectType.FILE fail("Found unsupported filesystem entry %s" % (fpath, )) def git_to_type(perm: str) -> ObjectType: """Get type of given Git entry from permission mode.""" if perm == "40000": return ObjectType.DIR elif perm == "120000": return ObjectType.LINK elif perm == "100755": return ObjectType.EXEC elif perm == "100644": return ObjectType.FILE fail("Cannot assign object type for entry with permission code %s" % (perm, )) def get_tree_raw_id(source_dir: str, repo_root: str) -> bytes: """Write the content of the directory recursively to the given repository and return its SHA1 hash and its raw bytes representation.""" entries: Dict[str, Tuple[bytes, ObjectType]] = {} for fname in os.listdir(source_dir): fpath = source_dir + "/" + fname obj_type = path_to_type(fpath) raw_h: bytes = b"" if obj_type == ObjectType.DIR: raw_h = get_tree_raw_id(fpath, repo_root) fname = fname + '/' # trailing '/' added for correct sorting elif obj_type == ObjectType.LINK: data = os.readlink(fpath).encode('utf-8') raw_h = write_blob_to_repo(repo_root, data) else: with open(fpath, "rb") as f: data = f.read() raw_h = write_blob_to_repo(repo_root, data) # Add entry to map entries[fname] = (raw_h, obj_type) return write_tree_to_repo(repo_root, entries) def import_to_git(target: str, *, repo_type: str, content_id: str, fail_context: str) -> str: """Import directory into Git cache and return its Git-tree identifier.""" fail_context += "While importing to Git directory %s:\n" % (target, ) # In order to import content that might otherwise be ignored by Git, such # as empty directories or magic-named files and folders (e.g., .git, # .gitignore), add entries manually to the repository, which should be in # its own separate location. repo_tmp_dir = create_tmp_dir(type="import-to-git") # Initialize repo to have access to its storage run_cmd(g_LAUNCHER + [g_GIT, "init"], cwd=repo_tmp_dir, fail_context=fail_context) # Get tree id of added directory try: tree_id: str = get_tree_raw_id(target, repo_tmp_dir).hex() except Exception as ex: fail(fail_context + "Writing tree to temporary repository failed with:\n%r" % (ex, )) # Commit the tree git_env = {**os.environ.copy(), **GIT_NOBODY_ENV} commit: str = run_cmd(g_LAUNCHER + [ g_GIT, "commit-tree", tree_id, "-m", "Content of %s %r" % (repo_type, content_id) ], stdout=subprocess.PIPE, cwd=repo_tmp_dir, env=git_env, fail_context=fail_context)[0].decode('utf-8').strip() # Update the HEAD to make the tree fetchable run_cmd(g_LAUNCHER + [g_GIT, "update-ref", "HEAD", commit], cwd=repo_tmp_dir, env=git_env, fail_context=fail_context) # Fetch commit into Git cache repository and tag it ensure_git_init(upstream=None, fail_context=fail_context) git_fetch(from_repo=repo_tmp_dir, to_repo=None, fetchable="", fail_context=fail_context) git_keep(commit, upstream=None, fail_context=fail_context) return tree_id def git_tree(*, commit: str, subdir: str, upstream: Optional[str], fail_context: str) -> str: """Get Git-tree identifier based on commit. Fails if the commit is not part of the repository. It is a user error if the referenced Git repository does not exist.""" tree = run_cmd(["git", "log", "-n", "1", "--format=%T", commit], stdout=subprocess.PIPE, cwd=git_root(upstream=upstream), fail_context=fail_context)[0].decode('utf-8').strip() return git_subtree(tree=tree, subdir=subdir, upstream=upstream, fail_context=fail_context) def git_subtree(*, tree: str, subdir: str, upstream: Optional[str], fail_context: str) -> str: """Get Git-tree identifier in a Git tree by subdirectory path. Fails if the tree is not part of the repository. It is a user error if the referenced Git repository does not exist.""" if os.path.normpath(subdir) == ".": return tree return run_cmd( g_LAUNCHER + [g_GIT, "rev-parse", "%s:%s" % (tree, os.path.normpath(subdir))], stdout=subprocess.PIPE, cwd=git_root(upstream=upstream), fail_context=fail_context, )[0].decode('utf-8').strip() def try_read_object_from_repo(obj_id: str, obj_type: str, *, upstream: Optional[str]) -> Optional[bytes]: """Return raw (binary) content of object referenced by identifier and type if object is in given Git repository, or None otherwise. Does not require the repository to exist, in which case it returns None. Expected obj_type values match those of cat-file: 'blob', 'tree', 'commit', 'tag'.""" root: str = git_root(upstream=upstream) if not os.path.exists(root): return None result = run_cmd(g_LAUNCHER + [g_GIT, "cat-file", obj_type, obj_id], stdout=subprocess.PIPE, cwd=root, fail_context=None) return result[0] if result[1] == 0 else None def read_git_tree(tree_id: str, *, upstream: Optional[str], fail_context: str) -> Dict[str, Tuple[bytes, ObjectType]]: """Reads a Git tree and returns a list of its entries. Tree entries have as key their filename and as value a tuple of raw id and object type. Method fails if the given tree is not part of the repository. It is a user error if the referenced Git repository does not exist.""" raw_tree_content = try_read_object_from_repo(tree_id, "tree", upstream=upstream) if raw_tree_content is None: fail(fail_context + "Failed to read Git tree %s from %s" % (tree_id, git_root(upstream=upstream))) # Parse the raw content; the Git tree format is: # " \0[next entries...]" # The hash size for SHA1 is 20 bytes entries: Dict[str, Tuple[bytes, ObjectType]] = {} curr_index = 0 while curr_index < len(raw_tree_content): # get permission perm_step = raw_tree_content[curr_index:].find(b' ') perm: str = raw_tree_content[curr_index:curr_index + perm_step].decode('utf-8') curr_index += perm_step + 1 # get filename name_step = raw_tree_content[curr_index:].find(b'\0') filename: str = raw_tree_content[curr_index:curr_index + name_step].decode('utf-8') curr_index += name_step + 1 # get raw id raw_id: bytes = raw_tree_content[curr_index:curr_index + SHA1_SIZE_BYTES].hex().encode('utf-8') curr_index += SHA1_SIZE_BYTES # store current entry entries[filename] = (raw_id, git_to_type(perm)) return entries ### # CAS utils ## def gc_storage_lock_acquire(is_shared: bool = False) -> TextIO: """Acquire garbage collector file lock for the local storage.""" # use same naming scheme as in Just return lock_acquire(os.path.join(g_ROOT, "protocol-dependent", "gc.lock"), is_shared) def git_hash(content: bytes, type: str = "blob") -> Tuple[str, bytes]: """Hash content as a Git object. Returns the hash, as well as the header to be stored.""" header = "{} {}\0".format(type, len(content)).encode('utf-8') h = hashlib.sha1() h.update(header) h.update(content) return h.hexdigest(), header def add_to_cas(data: Union[str, bytes]) -> Tuple[str, str]: """Add content to local file CAS and return its CAS location and hash.""" try: if isinstance(data, str): data = data.encode('utf-8') h, _ = git_hash(data) cas_root = os.path.join( g_ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}") basename = h[2:] target = os.path.join(cas_root, basename) tempname = os.path.join(cas_root, "%s.%d" % (basename, os.getpid())) if os.path.exists(target): return target, h os.makedirs(cas_root, exist_ok=True) with open(tempname, "wb") as f: f.write(data) f.flush() os.chmod(f.fileno(), 0o444) os.fsync(f.fileno()) os.utime(tempname, (0, 0)) os.rename(tempname, target) return target, h except Exception as ex: fail("Adding content to CAS failed with:\n%r" % (ex, )) def cas_path(h: str) -> str: """Get path to local file CAS.""" return os.path.join( g_ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}", h[2:]) def is_in_cas(h: str) -> bool: """Check if content is in local file CAS.""" return os.path.exists(cas_path(h)) ### # Staging utils ## def stage_git_entry(*, fpath: str, obj_id: str, obj_type: ObjectType, upstream: Optional[str], fail_context: str) -> None: """Stage specified Git entry, identified by id and object type, to a given location. It is a user error if the referenced Git repository does not exist.""" curr_fail_context = fail_context + "While staging entry %r:\n" % ( json.dumps({fpath: (obj_id, type_to_string(obj_type))}), ) # Trees need to get traversed if obj_type == ObjectType.DIR: os.makedirs(fpath) entries = read_git_tree(obj_id, upstream=upstream, fail_context=curr_fail_context) for key, val in entries.items(): stage_git_entry( fpath=os.path.join(fpath, key), obj_id=val[0].decode('utf-8'), obj_type=val[1], upstream=upstream, fail_context=fail_context, # limit log verbosity ) # Blobs are read as-is; only do work for supported blob types elif obj_type in SUPPORTED_BLOB_TYPES: content = try_read_object_from_repo(obj_id, "blob", upstream=upstream) if content is None: fail(curr_fail_context + "Failed to read Git entry!") try: if obj_type == ObjectType.LINK: os.symlink(src=content.decode('utf-8'), dst=fpath) else: with open(fpath, "wb") as f: fstat = os.stat(f.fileno()) f.write(content) f.flush() if obj_type == ObjectType.EXEC: os.chmod(f.fileno(), fstat.st_mode | stat.S_IEXEC) os.fsync(f.fileno()) except OSError: fail(curr_fail_context + "Failed to write entry") except Exception as ex: fail(curr_fail_context + "Writing entry failed with:\n%r" % (ex, )) else: # Warn if any unsupported entries were found warn(curr_fail_context + "Skipped staging of entry with unsupported type") return def stage_git_commit(commit: str, *, upstream: Optional[str], stage_to: str, fail_context: str) -> None: """Stage into a given directory the tree of a commit from given repository. Fails if the commit is not part of the repository. It is a user error if the referenced Git repository does not exist.""" fail_context += "While trying to stage commit %s\n" % (commit, ) # Stage underlying Git tree tree = git_tree(commit=commit, subdir=".", upstream=upstream, fail_context=fail_context) entries = read_git_tree(tree, upstream=upstream, fail_context=fail_context) os.makedirs(stage_to, exist_ok=True) # root dir can already exist for key, val in entries.items(): stage_git_entry(fpath=os.path.join(stage_to, key), obj_id=val[0].decode('utf-8'), obj_type=val[1], upstream=upstream, fail_context=fail_context) ### # Imports utils ## def get_repo_to_import(config: Json) -> str: """From a given repository config, take the main repository.""" main = config.get("main") if main is not None: if not isinstance(main, str): fail("Foreign config contains malformed \"main\" field:\n%r" % (json.dumps(main, indent=2), )) return main repos = config.get("repositories", {}) if repos and isinstance(repos, dict): # take main repo as first named lexicographically return sorted(repos.keys())[0] fail("Config does not contain any repositories; unsure what to import") def get_base_repo_if_computed(repo: Any, repos_config: Json) -> Optional[str]: """If repository is computed, return the base repository name.""" while isinstance(repo, str): repo = repos_config[repo]["repository"] if repo.get("type") in ["computed", "tree structure"]: return cast(str, repo.get("repo")) return None def repos_to_import(repos_config: Json, entry: str, known: Set[str]) -> Tuple[List[str], List[str]]: """Compute the set of transitively reachable repositories and the collection of repositories additionally needed as they serve as layers for the repositories to import.""" to_import: Set[str] = set() extra_imports: Set[str] = set() def visit(name: str) -> None: # skip any existing or already visited repositories if name in known or name in to_import: return repo_desc: Json = repos_config.get(name, {}) # if proper import, visit bindings, which are fully imported if name not in extra_imports: to_import.add(name) vals = cast(Dict[str, str], repo_desc.get("bindings", {})).values() for n in vals: extra_imports.discard(n) visit(n) repo = repo_desc.get("repository") if isinstance(repo, str): # visit referred repository, but skip bindings if repo not in known and repo not in to_import: extra_imports.add(repo) visit(repo) else: # if computed, visit the referred repository repo_base = get_base_repo_if_computed(repo, repos_config) if repo_base is not None: extra_imports.discard(repo_base) visit(repo_base) # add layers as extra imports, but referred repositories of computed # layers need to be fully imported for layer in ALT_DIRS: if layer in repo_desc: extra: str = repo_desc[layer] if extra not in known and extra not in to_import: extra_imports.add(extra) extra_repo_base = get_base_repo_if_computed( repos_config.get(extra, {}).get("repository", {}), repos_config) if extra_repo_base is not None: extra_imports.discard(extra_repo_base) visit(extra_repo_base) visit(entry) return list(to_import), list(extra_imports) def name_imports(to_import: List[str], extra_imports: List[str], existing: Set[str], base_name: str, main: Optional[str] = None) -> Dict[str, str]: """Assign names to the repositories to import in such a way that no conflicts arise.""" assign: Dict[str, str] = {} def find_name(name: str) -> str: base: str = "%s/%s" % (base_name, name) if (base not in existing) and (base not in assign): return base count: int = 0 while True: count += 1 candidate: str = base + " (%d)" % count if (candidate not in existing) and (candidate not in assign): return candidate if main is not None and (base_name not in existing): assign[main] = base_name to_import = [x for x in to_import if x != main] extra_imports = [x for x in extra_imports if x != main] for repo in to_import + extra_imports: assign[repo] = find_name(repo) return assign def rewrite_file_repo(repo: Json, remote_type: str, remote_stub: Dict[str, Any], *, fail_context: str) -> Json: """Rewrite \"file\"-type descriptions based on remote type.""" if remote_type == "git": # for imports from Git, file repos become type 'git' with subdir; the # validity of the new subdir value is not checked changes = {} subdir: str = os.path.normpath(repo.get("path", ".")) if subdir != ".": changes["subdir"] = subdir # keep ignore special and absent pragmas pragma = {} if repo.get("pragma", {}).get("special", None) == "ignore": pragma["special"] = "ignore" if repo.get("pragma", {}).get("absent", False): pragma["absent"] = True if pragma: changes["pragma"] = pragma return dict(remote_stub, **changes) elif remote_type == "file": # for imports from local checkouts, file repos remain type 'file'; only # relative paths get updated; paths are not checked for validity changes = {} root: str = remote_stub["path"] path: str = os.path.normpath(repo.get("path", ".")) if not Path(path).is_absolute(): changes["path"] = os.path.join(root, path) return dict(repo, **changes) elif remote_type in ["archive", "zip"]: # for imports from archives, file repos become archive type with subdir; # any path is prepended by the subdir provided in the input file, # if any; the validity of the new subdir is not checked changes = {} subdir: str = os.path.normpath(repo.get("path", ".")) if subdir != ".": existing: str = os.path.normpath(remote_stub.get("subdir", ".")) if existing != ".": subdir = os.path.join(existing, subdir) changes["subdir"] = subdir # keep special and absent pragmas pragma = {} special: Json = repo.get("pragma", {}).get("special", None) if special: pragma["special"] = special if repo.get("pragma", {}).get("absent", False): pragma["absent"] = True if pragma: changes["pragma"] = pragma return dict(remote_stub, **changes) elif remote_type == "git tree": # for imports from git-trees, file repos become 'git tree' types; the # subtree Git identifier is computed relative to the root Git tree, so # compute and validate the subtree path based on the source tree subdir # passed in the remote stub; the final configuration must NOT have any # subdir field path = cast(str, repo.get("path", ".")) if Path(path).is_absolute(): fail( fail_context + "Cannot import transitive \"file\" dependency with absolute path %s" % (path, )) remote_desc = dict(remote_stub) # keep remote_stub read-only! root: str = remote_desc.pop("subdir", ".") # remove 'subdir' key subdir = os.path.normpath(os.path.join(root, path)) if subdir.startswith(".."): fail(fail_context + "Transitive \"file\" dependency requests upward subtree %s" % (subdir, )) if subdir != ".": # get the subtree Git identifier remote_desc["id"] = git_subtree(tree=remote_desc["id"], subdir=subdir, upstream=None, fail_context=fail_context) # keep ignore special and absent pragmas pragma = {} if repo.get("pragma", {}).get("special", None) == "ignore": pragma["special"] = "ignore" if repo.get("pragma", {}).get("absent", False): pragma["absent"] = True if pragma: remote_desc["pragma"] = pragma return remote_desc fail("Unsupported remote type!") def update_pragmas(repo: Json, import_pragma: Json, pragma_special: Optional[str]) -> Json: """Update the description with any input-provided pragmas: - for all repositories, merge with import-level "absent" pragma - for "file"-type repositories, merge with import-level "to_git" pragma - for all repositories, overwrite with source-level "special" pragma.""" existing: Json = dict(repo.get("pragma", {})) # operate on copy # all repos support "absent pragma" absent: bool = existing.get("absent", False) or import_pragma.get( "absent", False) if absent: existing["absent"] = True # support "to_git" pragma for "file"-type repos if repo.get("type") == "file": to_git = existing.get("to_git", False) or import_pragma.get( "to_git", False) if to_git: existing["to_git"] = True # all repos get the "special" pragma overwritten, if provided if pragma_special is not None: existing["special"] = pragma_special # all other pragmas as kept; if no pragma was set, do not set any if existing: repo = dict(repo, **{"pragma": existing}) return repo def rewrite_repo(repo_spec: Json, *, remote_type: str, remote_stub: Dict[str, Any], assign: Json, import_pragma: Json, pragma_special: Optional[str], as_layer: bool, fail_context: str) -> Json: """Rewrite description of imported repositories.""" new_spec: Json = {} repo = repo_spec.get("repository", {}) if isinstance(repo, str): repo = assign[repo] elif repo.get("type") == "file": # "file"-type repositories need to be rewritten based on remote type repo = rewrite_file_repo(repo, remote_type, remote_stub, fail_context=fail_context) elif repo.get("type") == "distdir": existing_repos: List[str] = repo.get("repositories", []) new_repos = [assign[k] for k in existing_repos] repo = dict(repo, **{"repositories": new_repos}) elif repo.get("type") in ["computed", "tree structure"]: target: str = repo.get("repo", None) repo = dict(repo, **{"repo": assign[target]}) # update pragmas, as needed if isinstance(repo, dict): repo = update_pragmas(repo, import_pragma, pragma_special) new_spec["repository"] = repo # rewrite other roots and bindings, if actually needed to be imported if not as_layer: for key in ["target_root", "rule_root", "expression_root"]: if key in repo_spec: new_spec[key] = assign[repo_spec[key]] for key in [ "target_file_name", "rule_file_name", "expression_file_name" ]: if key in repo_spec: new_spec[key] = repo_spec[key] bindings = repo_spec.get("bindings", {}) new_bindings = {} for k, v in bindings.items(): new_bindings[k] = assign[v] if new_bindings: new_spec["bindings"] = new_bindings return new_spec def handle_import(remote_type: str, remote_stub: Dict[str, Any], repo_desc: Json, core_repos: Json, foreign_config: Json, pragma_special: Optional[str], *, fail_context: str) -> Json: """General handling of repository import from a foreign config.""" fail_context += "While handling import from remote type \"%s\"\n" % ( remote_type, ) # parse input description import_as: Optional[str] = repo_desc.get("alias", None) if import_as is not None and not isinstance(import_as, str): fail( fail_context + "Expected \"repos\" entry subfield \"import_as\" to be a string, but found:\n%r" % (json.dumps(import_as, indent=2), )) foreign_name: Optional[str] = repo_desc.get("repo", None) if foreign_name is not None and not isinstance(foreign_name, str): fail( fail_context + "Expected \"repos\" entry subfield \"repo\" to be a string, but found:\n%r" % (json.dumps(foreign_name, indent=2), )) if foreign_name is None: # if not provided, get main repository from source config foreign_name = get_repo_to_import(foreign_config) import_map: Json = repo_desc.get("map", None) if import_map is None: import_map = {} elif not isinstance(import_map, dict): fail( fail_context + "Expected \"repos\" entry subfield \"map\" to be a map, but found:\n%r" % (json.dumps(import_map, indent=2), )) pragma: Json = repo_desc.get("pragma", None) if pragma is None: pragma = {} elif not isinstance(pragma, dict): fail( fail_context + "Expected \"repos\" entry subfield \"pragma\" to be a map, but found:\n%r" % (json.dumps(pragma, indent=2), )) # Handle import with renaming foreign_repos: Json = foreign_config.get("repositories", {}) if foreign_repos is None or not isinstance(foreign_repos, dict): fail( fail_context + "Found empty or malformed \"repositories\" field in source configuration file" ) main_repos, extra_imports = repos_to_import(foreign_repos, foreign_name, set(import_map.keys())) extra_repos = sorted([x for x in main_repos if x != foreign_name]) ordered_imports: List[str] = [foreign_name] + extra_repos extra_imports = sorted(extra_imports) import_name = import_as if import_as is not None else foreign_name assign: Dict[str, str] = name_imports(ordered_imports, extra_imports, set(core_repos.keys()), import_name, main=foreign_name) # Report progress report("Importing %r as %r" % (foreign_name, import_name)) report("\tTransitive dependencies to import: %r" % (extra_repos, )) report("\tRepositories imported as layers: %r" % (extra_imports, )) report(None) # adds newline total_assign = dict(assign, **import_map) new_repos = dict(core_repos) # avoid side-effects for repo in ordered_imports: new_repos[assign[repo]] = rewrite_repo(foreign_repos[repo], remote_type=remote_type, remote_stub=remote_stub, assign=total_assign, import_pragma=pragma, pragma_special=pragma_special, as_layer=False, fail_context=fail_context) for repo in extra_imports: new_repos[assign[repo]] = rewrite_repo(foreign_repos[repo], remote_type=remote_type, remote_stub=remote_stub, assign=total_assign, import_pragma=pragma, pragma_special=pragma_special, as_layer=True, fail_context=fail_context) return new_repos ### # Checkout utils ## class CheckoutInfo: """Stores the result of fetching and checking out source repositories.""" def __init__(self, srcdir: str, remote_stub: Json, to_clean_up: str): self.srcdir = srcdir """Sources directory""" self.remote_stub = remote_stub """Stub of remote configuration.""" self.to_clean_up = to_clean_up """Temporary directory to clean up after handling imports.""" ### # Import from Git ## def git_checkout(imports_entry: Json) -> Optional[CheckoutInfo]: """Fetch a given remote Git repository and checkout a specified branch. Return the checkout location, the repository description stub to use for rewriting 'file'-type dependencies, and the temp dir to later clean up.""" # Set granular logging message fail_context: str = "While checking out source \"git\":\n" # Get the repositories list repos: List[Any] = imports_entry.get("repos", []) if not isinstance(repos, list): fail(fail_context + "Expected field \"repos\" to be a list, but found:\n%r" % (json.dumps(repos, indent=2), )) # Check if anything is to be done if not repos: return None # Parse source fetch fields url: str = imports_entry.get("url", None) if not isinstance(url, str): fail(fail_context + "Expected field \"url\" to be a string, but found:\n%r" % (json.dumps(url, indent=2), )) branch: str = imports_entry.get("branch", None) if not isinstance(branch, str): fail(fail_context + "Expected field \"branch\" to be a string, but found:\n%r" % (json.dumps(branch, indent=2), )) commit: Optional[str] = imports_entry.get("commit", None) if commit is not None and not isinstance(commit, str): fail(fail_context + "Expected field \"commit\" to be a string, but found:\n%r" % (json.dumps(commit, indent=2), )) mirrors: List[str] = imports_entry.get("mirrors", []) if not isinstance(mirrors, list): fail(fail_context + "Expected field \"mirrors\" to be a list, but found:\n%r" % (json.dumps(mirrors, indent=2), )) inherit_env: List[str] = imports_entry.get("inherit env", []) if not isinstance(inherit_env, list): fail(fail_context + "Expected field \"inherit env\" to be a list, but found:\n%r" % (json.dumps(inherit_env, indent=2), )) # Fetch the source repository workdir: str = create_tmp_dir(type="git-checkout") srcdir: str = os.path.join(workdir, "src") if commit is None: # Get top commit of remote branch from definitive source location fetch_url = git_url_is_path(url) if fetch_url is None: fetch_url = url else: fetch_url = os.path.abspath(fetch_url) commit = run_cmd( g_LAUNCHER + [g_GIT, "ls-remote", fetch_url, branch], cwd=workdir, stdout=subprocess.PIPE, fail_context=fail_context)[0].decode('utf-8').split('\t')[0] if not git_commit_present(commit, upstream=None): # If commit not in Git cache repository, do shallow clone and get # HEAD commit from definitive source location report("\tFetching top commit from remote Git [%s]" % (url, )) run_cmd(g_LAUNCHER + [ g_GIT, "clone", "-b", branch, "--depth", "1", fetch_url, "src" ], cwd=workdir, fail_context=fail_context) # In the very small chance that the remote top commit changed in the # meanwhile, only trust what has been actually cloned commit = run_cmd( g_LAUNCHER + [g_GIT, "log", "-n", "1", "--pretty=%H"], cwd=srcdir, stdout=subprocess.PIPE, fail_context=fail_context)[0].decode('utf-8').strip() # Cache this commit by fetching it to Git cache and tagging it ensure_git_init(upstream=None, fail_context=fail_context) git_fetch(from_repo=srcdir, to_repo=None, fetchable=commit, fail_context=fail_context) git_keep(commit, upstream=None, fail_context=fail_context) else: report("\tCache hit for commit %s" % (commit, )) # Create checkout from commit in Git cache repository ensure_git_init(upstream=srcdir, init_bare=False, fail_context=fail_context) git_fetch(from_repo=None, to_repo=srcdir, fetchable=commit, fail_context=fail_context) run_cmd(g_LAUNCHER + [g_GIT, "checkout", commit], cwd=srcdir, fail_context=fail_context) else: if not git_commit_present(commit, upstream=None): # If commit not in Git cache repository, fetch witnessing branch # from remote into the Git cache repository. Try mirrors first, as # they are closer ensure_git_init(upstream=None, fail_context=fail_context) report("\tFetching commit %s from remote Git [%s]" % (commit, url)) fetched: bool = False for source in mirrors + [url]: if git_fetch(from_repo=source, to_repo=None, fetchable=branch, fail_context=None) and git_commit_present( commit, upstream=None): fetched = True break if not fetched: fail(fail_context + "Failed to fetch commit %s.\nTried locations:\n%s" % ( commit, "\n".join(["\t%s" % (x, ) for x in mirrors + [url]]), )) git_keep(commit, upstream=None, fail_context=fail_context) else: report("\tCache hit for commit %s" % (commit, )) # Create checkout from commit in Git cache repository ensure_git_init(upstream=srcdir, init_bare=False, fail_context=fail_context) git_fetch(from_repo=None, to_repo=srcdir, fetchable=commit, fail_context=fail_context) run_cmd(g_LAUNCHER + [g_GIT, "checkout", commit], cwd=srcdir, fail_context=fail_context) # Prepare the description stub used to rewrite "file"-type dependencies repo_stub: Dict[str, Any] = { "type": "git", "repository": url, "branch": branch, "commit": commit, } if mirrors: repo_stub = dict(repo_stub, **{"mirrors": mirrors}) if inherit_env: repo_stub = dict(repo_stub, **{"inherit env": inherit_env}) return CheckoutInfo(srcdir, repo_stub, workdir) def import_from_git(core_repos: Json, imports_entry: Json, checkout_info: CheckoutInfo) -> Json: """Handles imports from Git repositories. Requires the result of a call to git_checkout.""" # Set granular logging message fail_context: str = "While importing from source \"git\":\n" # Get needed known fields (validated during checkout) repos: List[Any] = imports_entry["repos"] # Parse remaining fields as_plain: Optional[bool] = imports_entry.get("as plain", False) if as_plain is not None and not isinstance(as_plain, bool): fail(fail_context + "Expected field \"as plain\" to be a bool, but found:\n%r" % (json.dumps(as_plain, indent=2), )) foreign_config_file: Optional[str] = imports_entry.get("config", None) if foreign_config_file is not None and not isinstance( foreign_config_file, str): fail(fail_context + "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) pragma_special: Optional[str] = imports_entry.get("pragma", {}).get("special", None) if pragma_special is not None and not isinstance(pragma_special, str): fail(fail_context + "Expected pragma \"special\" to be a string, but found:\n%r" % (json.dumps(pragma_special, indent=2), )) if not as_plain: # only enabled if as_plain is true pragma_special = None # Read in the foreign config file if foreign_config_file: foreign_config_file = os.path.join(checkout_info.srcdir, foreign_config_file) else: foreign_config_file = get_repository_config_file( DEFAULT_JUSTMR_CONFIG_NAME, checkout_info.srcdir) foreign_config: Json = {} if as_plain: foreign_config = {"main": "", "repositories": DEFAULT_REPO} else: if (foreign_config_file): try: with open(foreign_config_file) as f: foreign_config = json.load(f) except OSError: fail(fail_context + "Failed to open foreign config file %s" % (foreign_config_file, )) except Exception as ex: fail(fail_context + "Reading foreign config file failed with:\n%r" % (ex, )) else: fail(fail_context + "Failed to find the repository configuration file!") # Process the imported repositories, in order new_repos = dict(core_repos) # avoid side-effects for repo_entry in repos: if not isinstance(repo_entry, dict): fail(fail_context + "Expected \"repos\" entries to be objects, but found:\n%r" % (json.dumps(repo_entry, indent=2), )) repo_entry = cast(Json, repo_entry) new_repos = handle_import("git", checkout_info.remote_stub, repo_entry, new_repos, foreign_config, pragma_special, fail_context=fail_context) # Clean up local fetch try_rmtree(checkout_info.to_clean_up) return new_repos ### # Import from file ## def import_from_file(core_repos: Json, imports_entry: Json) -> Json: """Handles imports from a local checkout.""" # Set granular logging message fail_context: str = "While importing from source \"file\":\n" # Get the repositories list repos: List[Any] = imports_entry.get("repos", []) if not isinstance(repos, list): fail(fail_context + "Expected field \"repos\" to be a list, but found:\n%r" % (json.dumps(repos, indent=2), )) # Check if anything is to be done if not repos: # empty return core_repos # Parse source config fields path: str = imports_entry.get("path", None) if not isinstance(path, str): fail(fail_context + "Expected field \"path\" to be a string, but found:\n%r" % (json.dumps(path, indent=2), )) as_plain: Optional[bool] = imports_entry.get("as plain", False) if as_plain is not None and not isinstance(as_plain, bool): fail(fail_context + "Expected field \"as plain\" to be a bool, but found:\n%r" % (json.dumps(as_plain, indent=2), )) foreign_config_file: Optional[str] = imports_entry.get("config", None) if foreign_config_file is not None and not isinstance( foreign_config_file, str): fail(fail_context + "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) pragma_special: Optional[str] = imports_entry.get("pragma", {}).get("special", None) if pragma_special is not None and not isinstance(pragma_special, str): fail(fail_context + "Expected pragma \"special\" to be a string, but found:\n%r" % (json.dumps(pragma_special, indent=2), )) if not as_plain: # only enabled if as_plain is true pragma_special = None # Read in the foreign config file if foreign_config_file: foreign_config_file = os.path.join(path, foreign_config_file) else: foreign_config_file = get_repository_config_file( DEFAULT_JUSTMR_CONFIG_NAME, path) foreign_config: Json = {} if as_plain: foreign_config = {"main": "", "repositories": DEFAULT_REPO} else: if (foreign_config_file): try: with open(foreign_config_file) as f: foreign_config = json.load(f) except OSError: fail(fail_context + "Failed to open foreign config file %s" % (foreign_config_file, )) except Exception as ex: fail(fail_context + "Reading foreign config file failed with:\n%r" % (ex, )) else: fail(fail_context + "Failed to find the repository configuration file!") # Prepare the description stub used to rewrite "file"-type dependencies remote_stub: Dict[str, Any] = { "type": "file", "path": path, } # Process the imported repositories, in order new_repos = dict(core_repos) # avoid side-effects for repo_entry in repos: if not isinstance(repo_entry, dict): fail(fail_context + "Expected \"repos\" entries to be objects, but found:\n%r" % (json.dumps(repo_entry, indent=2), )) repo_entry = cast(Json, repo_entry) new_repos = handle_import("file", remote_stub, repo_entry, new_repos, foreign_config, pragma_special, fail_context=fail_context) return new_repos ### # Import from archive ## def archive_fetch(locations: List[str], *, content: Optional[str], sha256: Optional[str] = None, sha512: Optional[str] = None, fail_context: str) -> str: """Make sure an archive is available in local CAS. Try all the remote locations given. Return the content hash on success.""" if content is None or not is_in_cas(content): # If content is in Git cache, move to CAS and return success if content is not None: data = try_read_object_from_repo(content, "blob", upstream=None) if data is not None: _, content = add_to_cas(data) report("\tCache hit for archive %s" % (content, )) return content # Fetch from remote fetched: bool = False report("\tFetching archive from remote locations [%s]" % (locations[-1])) for source in locations: data, err_code = run_cmd(g_LAUNCHER + ["wget", "-O", "-", source], stdout=subprocess.PIPE, cwd=os.getcwd()) if err_code == 0: # Compare with checksums, if given if sha256 is not None: actual_hash = hashlib.sha256(data).hexdigest() if sha256 != actual_hash: continue if sha512 is not None: actual_hash = hashlib.sha512(data).hexdigest() if sha512 != actual_hash: continue # Add to CAS and compare with expected content, if given _, computed_hash = add_to_cas(data) if content is not None: if content != computed_hash: continue fetched = True break else: content = computed_hash fetched = True break if not fetched: fail(fail_context + "Failed to fetch archive.\nTried locations:\n%s" % ("\n".join(["\t%s" % (x, ) for x in locations]), )) return cast(str, content) def archive_fetch_with_parse(repository: Json, *, fail_context: str) -> str: """Utility on top of archive_fetch that does its own parsing. Returns the Git identifier of the fetched content.""" # Parse fields fetch: str = repository.get("fetch", None) if not isinstance(fetch, str): fail(fail_context + "Expected field \"fetch\" to be a string, but found:\n%r" % (json.dumps(fetch, indent=2), )) content: str = repository.get("content", None) if not isinstance(content, str): fail(fail_context + "Expected field \"content\" to be a string, but found:\n%r" % (json.dumps(content, indent=2), )) mirrors: List[str] = repository.get("mirrors", []) if not isinstance(mirrors, list): fail(fail_context + "Expected field \"mirrors\" to be a list, but found:\n%r" % (json.dumps(mirrors, indent=2), )) sha256: Optional[str] = repository.get("sha256", None) if sha256 is not None and not isinstance(sha256, str): fail(fail_context + "Expected field \"sha256\" to be a string, but found:\n%r" % (json.dumps(sha256, indent=2), )) sha512: Optional[str] = repository.get("sha512", None) if sha512 is not None and not isinstance(sha512, str): fail(fail_context + "Expected field \"sha512\" to be a string, but found:\n%r" % (json.dumps(sha512, indent=2), )) # Fetch the archive to local CAS archive_fetch(mirrors + [fetch], content=content, sha256=sha256, sha512=sha512, fail_context=fail_context) return content def unpack_archive(content_id: str, *, archive_type: str, unpack_to: str, fail_context: str) -> None: """Unpack archive stored as a local CAS blob into a given directory.""" fail_context += "While unpacking archive %s:\n" % (cas_path(content_id), ) # ensure destination path is valid if os.path.exists(unpack_to): if not os.path.isdir(unpack_to): fail(fail_context + "Unpack location %s exists and is not a directory!" % (unpack_to, )) if os.listdir(unpack_to): fail(fail_context + "Cannot unpack to nonempty directory %s" % (unpack_to, )) else: os.makedirs(unpack_to, exist_ok=True) # unpack based on archive type if archive_type == "zip": # try as zip and 7z archives if run_cmd(g_LAUNCHER + ["unzip", "-d", ".", cas_path(content_id)], cwd=unpack_to, fail_context=None)[1] != 0 and run_cmd( g_LAUNCHER + ["7z", "x", cas_path(content_id)], cwd=unpack_to, fail_context=None)[1] != 0: fail(fail_context + "Failed to extract zip-like archive %s" % (cas_path(content_id), )) else: # try as tarball if run_cmd(g_LAUNCHER + ["tar", "xf", cas_path(content_id)], cwd=unpack_to, fail_context=None)[1] != 0: fail(fail_context + "Failed to extract tarball %s" % (cas_path(content_id), )) return def archive_checkout(imports_entry: Json) -> Optional[CheckoutInfo]: """Fetch a given remote archive to local CAS, unpack it, check content, and return the checkout location.""" # Set granular logging message fail_context: str = "While checking out source \"archive\":\n" # Get the repositories list repos: List[Any] = imports_entry.get("repos", []) if not isinstance(repos, list): fail(fail_context + "Expected field \"repos\" to be a list, but found:\n%r" % (json.dumps(repos, indent=2), )) # Check if anything is to be done if not repos: return None # Parse source fetch fields fetch: str = imports_entry.get("fetch", None) if not isinstance(fetch, str): fail(fail_context + "Expected field \"fetch\" to be a string, but found:\n%r" % (json.dumps(fetch, indent=2), )) archive_type: str = "archive" # type according to 'just-mr' tmp_type: Optional[str] = imports_entry.get("type", None) if tmp_type is not None: if not isinstance(tmp_type, str): fail(fail_context + "Expected field \"type\" to be a string, but found:\n%r" % (json.dumps(tmp_type, indent=2), )) if tmp_type not in ["tar", "zip"]: # values expected in input file warn( fail_context + "Field \"type\" has unsupported value %r\nTrying with default value 'tar'" % (json.dumps(tmp_type), )) else: archive_type = "zip" if tmp_type == "zip" else "archive" content: Optional[str] = imports_entry.get("content", None) if content is not None and not isinstance(content, str): fail(fail_context + "Expected field \"content\" to be a string, but found:\n%r" % (json.dumps(content, indent=2), )) mirrors: List[str] = imports_entry.get("mirrors", []) if not isinstance(mirrors, list): fail(fail_context + "Expected field \"mirrors\" to be a list, but found:\n%r" % (json.dumps(mirrors, indent=2), )) sha256: Optional[str] = imports_entry.get("sha256", None) if sha256 is not None and not isinstance(sha256, str): fail(fail_context + "Expected field \"sha256\" to be a string, but found:\n%r" % (json.dumps(sha256, indent=2), )) sha512: Optional[str] = imports_entry.get("sha512", None) if sha512 is not None and not isinstance(sha512, str): fail(fail_context + "Expected field \"sha512\" to be a string, but found:\n%r" % (json.dumps(sha512, indent=2), )) subdir: Optional[str] = imports_entry.get("subdir", None) if subdir is not None: if not isinstance(subdir, str): fail(fail_context + "Expected field \"subdir\" to be a string, but found:\n%r" % (json.dumps(subdir, indent=2), )) subdir = os.path.normpath(subdir) if os.path.isabs(subdir) or subdir.startswith(".."): fail( fail_context + "Expected field \"subdir\" to be a relative non-upward path, but found:\n%r" % (json.dumps(subdir, indent=2), )) if subdir == ".": subdir = None # treat as if missing # Fetch the source repository if content is None: # If content is not known, get it from the definitive source location content = archive_fetch([fetch], content=None, sha256=sha256, sha512=sha512, fail_context=fail_context) else: # If content known, try the mirrors first, as they are closer archive_fetch(mirrors + [fetch], content=content, sha256=sha256, sha512=sha512, fail_context=fail_context) workdir: str = create_tmp_dir(type="archive-checkout") unpack_archive(content, archive_type=archive_type, unpack_to=workdir, fail_context=fail_context) srcdir = (workdir if subdir is None else os.path.join(workdir, subdir)) # Prepare the description stub used to rewrite "file"-type dependencies repo_stub: Dict[str, Any] = { "type": "zip" if archive_type == "zip" else "archive", "fetch": fetch, "content": content, } if mirrors: repo_stub = dict(repo_stub, **{"mirrors": mirrors}) if sha256 is not None: repo_stub = dict(repo_stub, **{"sha256": sha256}) if sha512 is not None: repo_stub = dict(repo_stub, **{"sha512": sha512}) if subdir is not None: repo_stub = dict(repo_stub, **{"subdir": subdir}) return CheckoutInfo(srcdir, repo_stub, workdir) def import_from_archive(core_repos: Json, imports_entry: Json, checkout_info: CheckoutInfo) -> Json: """Handles imports from archive-type repositories. Requires the result of a call to archive_checkout.""" # Set granular logging message fail_context: str = "While importing from source \"archive\":\n" # Get needed known fields (validated during checkout) repos: List[Any] = imports_entry["repos"] archive_type: str = imports_entry.get("type", "tar") if archive_type != "zip": archive_type = "archive" # type name as in Just-MR # Parse remaining fields as_plain: Optional[bool] = imports_entry.get("as plain", False) if as_plain is not None and not isinstance(as_plain, bool): fail(fail_context + "Expected field \"as plain\" to be a bool, but found:\n%r" % (json.dumps(as_plain, indent=2), )) foreign_config_file: Optional[str] = imports_entry.get("config", None) if foreign_config_file is not None and not isinstance( foreign_config_file, str): fail(fail_context + "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) pragma_special: Optional[str] = imports_entry.get("pragma", {}).get("special", None) if pragma_special is not None and not isinstance(pragma_special, str): fail(fail_context + "Expected pragma \"special\" to be a string, but found:\n%r" % (json.dumps(pragma_special, indent=2), )) if not as_plain: # only enabled if as_plain is true pragma_special = None # Read in the foreign config file if foreign_config_file: foreign_config_file = os.path.join(checkout_info.srcdir, foreign_config_file) else: foreign_config_file = get_repository_config_file( DEFAULT_JUSTMR_CONFIG_NAME, checkout_info.srcdir) foreign_config: Json = {} if as_plain: foreign_config = {"main": "", "repositories": DEFAULT_REPO} else: if (foreign_config_file): try: with open(foreign_config_file) as f: foreign_config = json.load(f) except OSError: fail(fail_context + "Failed to open foreign config file %s" % (foreign_config_file, )) except Exception as ex: fail(fail_context + "Reading foreign config file failed with:\n%r" % (ex, )) else: fail(fail_context + "Failed to find the repository configuration file!") # Process the imported repositories, in order new_repos = dict(core_repos) # avoid side-effects for repo_entry in repos: if not isinstance(repo_entry, dict): fail(fail_context + "Expected \"repos\" entries to be objects, but found:\n%r" % (json.dumps(repo_entry, indent=2), )) repo_entry = cast(Json, repo_entry) new_repos = handle_import(archive_type, checkout_info.remote_stub, repo_entry, new_repos, foreign_config, pragma_special, fail_context=fail_context) # Clean up local fetch try_rmtree(checkout_info.to_clean_up) return new_repos ### # Import from Git tree ## def git_tree_checkout(imports_entry: Json) -> Optional[CheckoutInfo]: """Run a given command or the command generated by the given command and import the obtained tree to Git cache. Return the checkout location, the repository description stub to use for rewriting 'file'-type dependencies, containing any additional needed data, and the temp dir to later clean up. """ # Set granular logging message fail_context: str = "While checking out source \"git tree\":\n" # Get the repositories list repos: List[Any] = imports_entry.get("repos", []) if not isinstance(repos, list): fail(fail_context + "Expected field \"repos\" to be a list, but found:\n%r" % (json.dumps(repos, indent=2), )) # Check if anything is to be done if not repos: return None # Parse source config fields command: Optional[List[str]] = imports_entry.get("cmd", None) if command is not None and not isinstance(command, list): fail(fail_context + "Expected field \"cmd\" to be a list, but found:\n%r" % (json.dumps(command, indent=2), )) command_gen: Optional[List[str]] = imports_entry.get("cmd gen", None) if command_gen is not None and not isinstance(command_gen, list): fail(fail_context + "Expected field \"cmd gen\" to be a list, but found:\n%r" % (json.dumps(command_gen, indent=2), )) if command is None == command_gen is None: fail(fail_context + "Only one of fields \"cmd\" and \"cmd gen\" must be provided!") subdir: Optional[str] = imports_entry.get("subdir", None) if subdir is not None: if not isinstance(subdir, str): fail(fail_context + "Expected field \"subdir\" to be a string, but found:\n%r" % (json.dumps(subdir, indent=2), )) subdir = os.path.normpath(subdir) if os.path.isabs(subdir) or subdir.startswith(".."): fail( fail_context + "Expected field \"subdir\" to be a relative non-upward path, but found:\n%r" % (json.dumps(subdir, indent=2), )) if subdir == ".": subdir = None # treat as if missing command_env: Json = imports_entry.get("env", {}) if not isinstance(command_env, dict): fail(fail_context + "Expected field \"env\" to be a map, but found:\n%r" % (json.dumps(command_env, indent=2), )) inherit_env: List[str] = imports_entry.get("inherit env", []) if not isinstance(inherit_env, list): fail(fail_context + "Expected field \"inherit env\" to be a list, but found:\n%r" % (json.dumps(inherit_env, indent=2), )) # Set the command environment curr_env = os.environ.copy() new_envs = {} for envar in inherit_env: if envar in curr_env: new_envs[envar] = curr_env[envar] command_env = dict(command_env, **new_envs) # Generate the command to be run, if needed report("\tGenerating Git-tree content") if command_gen is not None: tmpdir: str = create_tmp_dir( type="cmd-gen") # to avoid polluting the current dir data, _ = run_cmd(g_LAUNCHER + command_gen, cwd=tmpdir, env=command_env, stdout=subprocess.PIPE, fail_context=fail_context) tmp_cmd = json.loads(data) if not isinstance(tmp_cmd, list): fail( fail_context + "Generative command should have produced a list, but found:\n%r" % (data, )) command = tmp_cmd try_rmtree(tmpdir) # Generate the sources tree content; here we use the environment provided command = cast(List[str], command) workdir: str = create_tmp_dir(type="git-tree-checkout") run_cmd(g_LAUNCHER + command, cwd=workdir, env=command_env, fail_context=fail_context) # Import root tree to Git cache; as we do not have the tree hash, identify # commits by the hash of the generating command instead tree_id = import_to_git(workdir, repo_type="git tree", content_id=git_hash( json.dumps(command).encode('utf-8'))[0], fail_context=fail_context) # Point the sources path for imports to the right subdirectory srcdir = (workdir if subdir is None else os.path.join(workdir, subdir)) # Prepare the description stub used to rewrite "file"-type dependencies repo_stub: Dict[str, Any] = { "type": "git tree", "cmd": command, "env": command_env, # original env "id": tree_id, # the root tree id } if inherit_env: repo_stub = dict(repo_stub, **{"inherit env": inherit_env}) # The subdir should not be part of the final description, but is needed for # computing the subtree identifier if subdir is not None: repo_stub = dict(repo_stub, **{"subdir": subdir}) return CheckoutInfo(srcdir, repo_stub, workdir) def import_from_git_tree(core_repos: Json, imports_entry: Json, checkout_info: CheckoutInfo) -> Json: """Handles imports from general Git trees obtained by running a command (explicitly given or generated by a given command).""" # Set granular logging message fail_context: str = "While importing from source \"git tree\":\n" # Get needed known fields (validated during checkout) repos: List[Any] = imports_entry["repos"] # Parse remaining fields as_plain: Optional[bool] = imports_entry.get("as plain", False) if as_plain is not None and not isinstance(as_plain, bool): fail(fail_context + "Expected field \"as plain\" to be a bool, but found:\n%r" % (json.dumps(as_plain, indent=2), )) foreign_config_file: Optional[str] = imports_entry.get("config", None) if foreign_config_file is not None and not isinstance( foreign_config_file, str): fail(fail_context + "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) pragma_special: Optional[str] = imports_entry.get("pragma", {}).get("special", None) if pragma_special is not None and not isinstance(pragma_special, str): fail(fail_context + "Expected pragma \"special\" to be a string, but found:\n%r" % (json.dumps(pragma_special, indent=2), )) if not as_plain: # only enabled if as_plain is true pragma_special = None # Read in the foreign config file if foreign_config_file: foreign_config_file = os.path.join(checkout_info.srcdir, foreign_config_file) else: foreign_config_file = get_repository_config_file( DEFAULT_JUSTMR_CONFIG_NAME, checkout_info.srcdir) foreign_config: Json = {} if as_plain: foreign_config = {"main": "", "repositories": DEFAULT_REPO} else: if (foreign_config_file): try: with open(foreign_config_file) as f: foreign_config = json.load(f) except OSError: fail(fail_context + "Failed to open foreign config file %s" % (foreign_config_file, )) except Exception as ex: fail(fail_context + "Reading foreign config file failed with:\n%r" % (ex, )) else: fail(fail_context + "Failed to find the repository configuration file!") # Process the imported repositories, in order new_repos = dict(core_repos) # avoid side-effects for repo_entry in repos: if not isinstance(repo_entry, dict): fail(fail_context + "Expected \"repos\" entries to be objects, but found:\n%r" % (json.dumps(repo_entry, indent=2), )) repo_entry = cast(Json, repo_entry) new_repos = handle_import("git tree", checkout_info.remote_stub, repo_entry, new_repos, foreign_config, pragma_special, fail_context=fail_context) # Clean up local fetch try_rmtree(checkout_info.to_clean_up) return new_repos ### # Import from generic source ## def import_generic(core_config: Json, imports_entry: Json) -> Json: """Handles generic imports done via a user-defined command.""" # Set granular logging message fail_context: str = "While importing from source \"generic\":\n" # Parse source config fields command: List[str] = imports_entry.get("cmd", None) if not isinstance(command, list): fail(fail_context + "Expected field \"cmd\" to be a list, but found:\n%r" % (json.dumps(command, indent=2), )) command_env: Json = imports_entry.get("env", {}) if not isinstance(command_env, dict): fail(fail_context + "Expected field \"env\" to be a map, but found:\n%r" % (json.dumps(command_env, indent=2), )) inherit_env: List[str] = imports_entry.get("inherit env", []) if not isinstance(inherit_env, list): fail(fail_context + "Expected field \"inherit env\" to be a list, but found:\n%r" % (json.dumps(inherit_env, indent=2), )) command_cwd: str = imports_entry.get("cwd", os.getcwd()) if not isinstance(command_cwd, str): fail(fail_context + "Expected field \"cwd\" to be a string, but found:\n%r" % (json.dumps(command_cwd, indent=2), )) if not os.path.isabs(command_cwd): command_cwd = os.path.join(os.getcwd(), command_cwd) # Set the command environment curr_env = os.environ.copy() new_envs = {} for envar in inherit_env: if envar in curr_env: new_envs[envar] = curr_env[envar] command_env.update(new_envs) # Run the command command_output = run_cmd( g_LAUNCHER + command, env=command_env, stdout=subprocess.PIPE, input=json.dumps(core_config).encode('utf-8'), cwd=command_cwd, fail_context=fail_context)[0].decode('utf-8').strip() # Parse output as JSON and do minimal validation parsed_output: Json = {} try: parsed_output = json.loads(command_output) except Exception as ex: fail(fail_context + "Parsing output of command as JSON failed with:\n%r" % (ex, )) # Perform minimal validation and restrict keys to those expected if "repositories" not in parsed_output.keys(): fail(fail_context + "Output configuration is missing mandatory key \"repositories\"") new_config: Json = {"repositories": parsed_output["repositories"]} main: str = parsed_output.get("main", None) if main is not None: if not isinstance(main, str): fail( fail_context + "Output configuration has malformed value %s for key \"main\"" % (main, )) new_config["main"] = main return new_config ### # Cloning logic ## def rewrite_cloned_repo(repos: Json, *, clone_to: str, target_repo: str, ws_root_repo: str) -> Json: """Rewrite description of a locally-cloned repository.""" # For Git repositories, the clone always contains the whole root tree, # so the path will need to point to any given subdir; no validation of # fields is needed, as it was already done pre-cloning ws_root_desc: Json = repos[ws_root_repo] if ws_root_desc["repository"]["type"] == "git": subdir: Optional[str] = ws_root_desc["repository"].get("subdir", None) if subdir is not None and os.path.normpath(subdir) != ".": clone_to = os.path.join(clone_to, subdir) # Set workspace root description new_spec: Json = { "repository": { "type": "file", "path": os.path.abspath(clone_to) } } # Keep relevant pragmas from the workspace root repository pragma: Json = {} existing: Json = repos[ws_root_repo]["repository"].get("pragma", {}) special = existing.get("special", None) if special: pragma["special"] = special to_git = existing.get("to_git", False) if to_git: pragma["to_git"] = True if pragma: new_spec["repository"]["pragma"] = pragma # Keep bindings, roots, and root files from the target repository to be able # to build against it layer_desc: Json = repos[target_repo] for key in REPO_KEYS_TO_KEEP: if key in layer_desc: new_spec[key] = layer_desc[key] return new_spec def clone_repo(repos: Json, known_repo: str, deps_chain: List[str], clone_to: str) -> Optional[Tuple[str, str]]: """Clone the workspace root of a Git repository into a given directory path. The repository is described by a dependency chain from a given start repository. Returns the names of the target repository and of the repository providing its workspace root (which can be the same) on success, None if no cloning can be done.""" # Set granular logging message fail_context: str = "While processing clone entry %r:\n" % (json.dumps( {clone_to: [known_repo, deps_chain]}), ) # Check cloning path clone_to = os.path.abspath(clone_to) if os.path.exists(clone_to): if not os.path.isdir(clone_to): fail(fail_context + "Clone path %r exists and is not a directory!" % (json.dumps(clone_to), )) if os.listdir(clone_to): warn(fail_context + "Clone directory %r exists and is not empty! Skipping" % (json.dumps(clone_to), )) return None ## Helper functions def process_file(repository: Json, *, clone_to: str, fail_context: str) -> str: """Process a file repository and return its path.""" # Parse fields fpath: str = repository.get("path", None) if not isinstance(fpath, str): fail(fail_context + "Expected field \"path\" to be a string, but found:\n%r" % (json.dumps(fpath, indent=2), )) # Simply copy directory tree in the new location try: shutil.copytree(fpath, clone_to, symlinks=True, dirs_exist_ok=True) except Exception as ex: fail(fail_context + "Copying file path %s failed with:\n%r" % (fpath, ex)) return fpath def process_git(repository: Json, *, clone_to: str, fail_context: str) -> str: """Process a Git repository and return its commit id.""" # Parse fields commit: str = repository.get("commit", None) if not isinstance(commit, str): fail(fail_context + "Expected field \"commit\" to be a string, but found:\n%r" % (json.dumps(commit, indent=2), )) # If commit in Git cache, stage it and return if git_commit_present(commit, upstream=None): report("\tCache hit for commit %s" % (commit, )) stage_git_commit(commit, upstream=None, stage_to=clone_to, fail_context=fail_context) return commit # Parse fields needed for url: str = repository.get("repository", None) if not isinstance(url, str): fail(fail_context + "Expected field \"url\" to be a string, but found:\n%r" % (json.dumps(url, indent=2), )) branch: str = repository.get("branch", None) if not isinstance(branch, str): fail(fail_context + "Expected field \"branch\" to be a string, but found:\n%r" % (json.dumps(branch, indent=2), )) mirrors: List[str] = repository.get("mirrors", []) if not isinstance(mirrors, list): fail(fail_context + "Expected field \"mirrors\" to be a list, but found:\n%r" % (json.dumps(mirrors, indent=2), )) # Clone the branch fully and reset to specific commit; in this case, we # are interested also in the Git history, so a simple commit checkout is # not enough; try mirrors first, as they are closer report("\tCloning commit %s from remote" % (commit, )) cloned: bool = False sources = mirrors + [url] for source in sources: if (run_cmd(g_LAUNCHER + [g_GIT, "clone", "-b", branch, source, clone_to], cwd=os.getcwd(), fail_context=None)[1] == 0 and run_cmd(g_LAUNCHER + [g_GIT, "reset", "--hard", commit], cwd=clone_to, fail_context=None)[1] == 0): cloned = True break if not cloned: fail(fail_context + "Failed to clone Git repository.\nTried locations:\n%s" % ("\n".join(["\t%s" % (x, ) for x in sources]), )) return commit def process_archive(repository: Json, repo_type: str, *, clone_to: str, fail_context: str) -> str: """Process an archive-like repository and return its content id.""" # Parse entries not covered in fetch method to check for early fail subdir: Optional[str] = repository.get("subdir", None) if subdir is not None: if not isinstance(subdir, str): fail( fail_context + "Expected field \"subdir\" to be a string, but found:\n%r" % (json.dumps(subdir, indent=2), )) subdir = os.path.normpath(subdir) if os.path.isabs(subdir) or subdir.startswith(".."): fail( fail_context + "Expected field \"subdir\" to be a relative non-upward path, but found:\n%r" % (json.dumps(subdir, indent=2), )) if subdir == ".": subdir = None # treat as if missing # Fetch the archive content = archive_fetch_with_parse(repository, fail_context=fail_context) # Stage the content of the relevant subdir if subdir is None: # Unpack directly to clone location unpack_archive(content, archive_type=repo_type, unpack_to=clone_to, fail_context=fail_context) else: # Unpack to a temporary dir workdir: str = create_tmp_dir(type="archive-unpack") unpack_archive(content, archive_type=repo_type, unpack_to=workdir, fail_context=fail_context) # Keep relevant subdir move_from_dir: str = os.path.join(workdir, subdir) os.makedirs(clone_to, exist_ok=True) for entry in os.listdir(move_from_dir): # shutil.move uses os.rename if on same filesystem or # shutil.copy2 otherwise, which preserves file metadata # and uses hardlinks if supported by the OS try: shutil.move(os.path.join(move_from_dir, entry), clone_to) except Exception as ex: fail(fail_context + "Moving file path %s failed with:\n%r" % (os.path.join(move_from_dir, entry), ex)) # Clean up tmp dir try_rmtree(workdir) return content def process_foreign_file(repository: Json, *, clone_to: str, fail_context: str) -> str: """Process a foreign-file repository and return its content id.""" # Parse fields not in common with archive-type repositories name: str = repository.get("name", None) if not isinstance(name, str): fail(fail_context + "Expected field \"name\" to be a string, but found:\n%r" % (json.dumps(name, indent=2), )) exec: Optional[bool] = repository.get("executable", None) if exec is not None and not isinstance(exec, bool): fail(fail_context + "Expected field \"exec\" to be a boolean, but found:\n%r" % (json.dumps(exec, indent=2), )) # Fetch the foreign file to local CAS content = archive_fetch_with_parse(repository, fail_context=fail_context) # stage the file under the provided name in the clone directory os.makedirs(clone_to, exist_ok=True) abs_name = os.path.join(clone_to, name) shutil.copyfile(cas_path(content), abs_name) if exec == True: os.chmod(abs_name, os.stat(abs_name).st_mode | stat.S_IEXEC) return content def process_distdir(repository: Json, repos: Json, *, clone_to: str, fail_context: str) -> str: """Process a distdir repository and return its content tree id.""" # Helper method def get_distfile(desc: Json, *, fail_context: str) -> str: distfile: str = desc.get("distfile", None) if distfile is None: fetch: str = desc.get("fetch", None) if not isinstance(fetch, str): fail( fail_context + "Expected field \"fetch\" to be a string, but found:\n%r" % (json.dumps(fetch, indent=2), )) distfile = os.path.basename(cast(str, desc.get("fetch"))) return distfile # Parse fields distdir_list: List[str] = repository.get("repositories", None) if not isinstance(distdir_list, list): fail( fail_context + "Expected field \"repositories\" to be a list, but found:\n%r" % (json.dumps(distdir_list, indent=2), )) # Gather the distdirs content: Dict[str, str] = {} to_fetch: List[str] = [] for repo in distdir_list: repo_fail_context: str = fail_context + ( "While processing distdir entry \"%s\"" % (repo, )) # If repo does not exist, fail if repo not in repos: fail(repo_fail_context + "Distdir repository not found") repo_desc: Json = repos[repo].get("repository", {}) repo_desc_type = repo_desc.get("type") # Only do work for archived types if repo_desc_type in ["archive", "zip"]: content_id: str = repo_desc.get("content", None) if not isinstance(content_id, str): fail( repo_fail_context + "Expected field \"content\" to be a list, but found:\n%r" % (json.dumps(content_id, indent=2), )) # Store distfile-to-content map and witnessing repository entry content[get_distfile( repo_desc, fail_context=repo_fail_context)] = content_id to_fetch.append(repo) # Ensure distfiles are in CAS for repo in to_fetch: repo_fail_context: str = fail_context + ( "While processing distdir entry \"%s\"" % (repo, )) repo_desc: Json = repos[repo].get("repository", {}) archive_fetch_with_parse(repo_desc, fail_context=repo_fail_context) # Stage the distfiles os.makedirs(clone_to, exist_ok=True) for name, content_id in content.items(): abs_name = os.path.join(clone_to, name) shutil.copyfile(cas_path(content_id), abs_name) # Hash the content map as unique id for the distdir repo entry distdir_tree_id, _ = git_hash( json.dumps(content, sort_keys=True, separators=(',', ':')).encode('utf-8')) return distdir_tree_id def process_git_tree(repository: Json, *, clone_to: str, fail_context: str) -> str: """Process a git tree repository and return the tree id.""" # Parse tree id tree_id: str = repository.get("id", None) if not isinstance(tree_id, str): fail(fail_context + "Expected field \"id\" to be a string, but found:\n%r" % (json.dumps(tree_id, indent=2), )) # If tree is in Git cache, stage it from there if (try_read_object_from_repo(tree_id, "tree", upstream=None) is not None): report("\tCache hit for Git-tree %s" % (tree_id, )) os.makedirs(clone_to, exist_ok=True) stage_git_entry(fpath=clone_to, obj_id=tree_id, obj_type=ObjectType.DIR, upstream=None, fail_context=fail_context) return tree_id # Parse the other needed fields command: List[str] = repository.get("cmd", None) if not isinstance(command, list): fail(fail_context + "Expected field \"cmd\" to be a list, but found:\n%r" % (json.dumps(command, indent=2), )) command_env: Json = repository.get("env", {}) if not isinstance(command_env, dict): fail(fail_context + "Expected field \"env\" to be a map, but found:\n%r" % (json.dumps(command_env, indent=2), )) inherit_env: List[str] = repository.get("inherit env", []) if not isinstance(inherit_env, list): fail(fail_context + "Expected field \"inherit env\" to be a list, but found:\n%r" % (json.dumps(inherit_env, indent=2), )) # Get the actual command environment curr_env = os.environ.copy() new_envs = {} for envar in inherit_env: if envar in curr_env: new_envs[envar] = curr_env[envar] command_env = dict(command_env, **new_envs) # Generate the content report("\tGenerating Git-tree content") os.makedirs(clone_to, exist_ok=True) run_cmd(g_LAUNCHER + command, cwd=clone_to, env=command_env, fail_context=fail_context) # Cache the content; to not pollute the clone folder, do it via a # temporary location workdir: str = create_tmp_dir(type="git-tree-checkout") try: shutil.copytree(clone_to, workdir, symlinks=True, dirs_exist_ok=True) except Exception as ex: fail(fail_context + "Copying file path %s failed with:\n%r" % (clone_to, ex)) imported_tree_id = import_to_git(workdir, repo_type="git-tree", content_id=tree_id, fail_context=fail_context) if imported_tree_id != tree_id: # Allow, but give warning warn( fail_context + "Tree mismatch in \"git tree\" repository: expected %s, got %s\n" % (tree_id, imported_tree_id)) try_rmtree(workdir) # Always report the id of the actual content return imported_tree_id def follow_binding(repos: Json, *, repo_name: str, dep_name: str, fail_context: str) -> str: """Follow a named binding.""" if repo_name not in repos.keys(): fail(fail_context + "Failed to find repository %r" % (json.dumps(repo_name), )) if "bindings" not in repos[repo_name].keys(): fail(fail_context + "Repository %r does not have bindings!" % (json.dumps(repo_name), )) if dep_name not in repos[repo_name]["bindings"].keys(): fail(fail_context + "Repository %r does not have binding %r" % (json.dumps(repo_name), json.dumps(dep_name))) return repos[repo_name]["bindings"][dep_name] ## Main logic # Follow the bindings target_repo: str = known_repo for dep in deps_chain: target_repo = follow_binding(repos, repo_name=target_repo, dep_name=dep, fail_context=fail_context) if target_repo not in repos.keys(): fail(fail_context + "Failed to find repository %r" % (json.dumps(target_repo), )) # Get the workspace root of the target repository repo_to_clone: str = target_repo while isinstance(repos[repo_to_clone].get("repository", None), str): repo_to_clone = repos[repo_to_clone]["repository"] if repo_to_clone not in repos.keys(): fail(fail_context + "Failed to find repository %r" % (json.dumps(repo_to_clone), )) repo_desc: Json = repos[repo_to_clone] repository: Json = repo_desc["repository"] repo_type: str = repository.get("type", None) if not isinstance(repo_type, str): fail(fail_context + "Expected field \"type\" to be a string, but found:\n%r" % (json.dumps(repo_type, indent=2), )) # Clone the repository locally, based on type; lock exclusively for writing lockfile = lock_acquire(os.path.join(Path(clone_to).parent, "clone.lock")) report("Cloning workspace root of repository %r to %s" % (json.dumps(target_repo), clone_to)) result: Optional[Tuple[str, str]] = (target_repo, repo_to_clone) if repo_type == "file": fpath = process_file(repository, clone_to=clone_to, fail_context=fail_context) report("\tCloned file path %s to %s" % (fpath, clone_to)) elif repo_type == "git": commit = process_git(repository, clone_to=clone_to, fail_context=fail_context) report("\tCloned Git commit %s to %s" % (commit, clone_to)) elif repo_type in ["archive", "zip"]: content = process_archive(repository, repo_type, clone_to=clone_to, fail_context=fail_context) report("\tCloned archive-like content %s to %s" % (content, clone_to)) elif repo_type == "foreign file": content_id = process_foreign_file(repository, clone_to=clone_to, fail_context=fail_context) report("\tCloned foreign file %s to %s" % (content_id, clone_to)) elif repo_type == "distdir": content_id = process_distdir(repository, repos, clone_to=clone_to, fail_context=fail_context) report("\tCloned distdir %s to %s" % (content_id, clone_to)) elif repo_type == "git tree": tree_id = process_git_tree(repository, clone_to=clone_to, fail_context=fail_context) report("\tCloned git tree %s to %s" % (tree_id, clone_to)) elif repo_type in ["computed", "tree structure"]: warn(fail_context + "Cloning not supported for type %r. Skipping" % (json.dumps(repo_type), )) result = None else: warn(fail_context + "Found unknown type %r. Skipping" % (json.dumps(repo_type), )) result = None # Release lock and return keep list lock_release(lockfile) return result ### # Deduplication logic ## def bisimilar_repos(repos: Json) -> List[List[str]]: """Compute the maximal bisimulation between the repositories and return the bisimilarity classes.""" bisim: Dict[Tuple[str, str], Json] = {} def is_different(name_a: str, name_b: str) -> bool: pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) return bisim.get(pos, {}).get("different", False) def mark_as_different(name_a: str, name_b: str): nonlocal bisim pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) entry = bisim.get(pos, {}) if entry.get("different"): return bisim[pos] = dict(entry, **{"different": True}) also_different: List[Tuple[str, str]] = entry.get("different_if", []) for a, b in also_different: mark_as_different(a, b) def register_dependency(name_a: str, name_b: str, dep_a: str, dep_b: str): pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) entry = bisim.get(pos, {}) deps: List[Tuple[str, str]] = entry.get("different_if", []) deps.append((dep_a, dep_b)) bisim[pos] = dict(entry, **{"different_if": deps}) def roots_equal(a: Json, b: Json, name_a: str, name_b: str) -> bool: """Get equality condition between roots.""" if a["type"] != b["type"]: return False if a["type"] == "file": return a["path"] == b["path"] elif a["type"] in ["archive", "zip"]: return (a["content"] == b["content"] and a.get("subdir", ".") == b.get("subdir", ".")) elif a["type"] == "git": return (a["commit"] == b["commit"] and a.get("subdir", ".") == b.get("subdir", ".")) elif a["type"] in ["computed", "tree structure"]: if (a["type"] == "computed" and (a.get("config", {}) != b.get("config", {}) or a["target"] != b["target"])): return False if a["repo"] == b["repo"]: return True elif is_different(a["repo"], b["repo"]): return False else: # equality pending target repo equality register_dependency(a["repo"], b["repo"], name_a, name_b) return True else: # unknown repository type, the only safe way is to test # for full equality return a == b def get_root(repos: Json, name: str, *, root_name: str = "repository", default_root: Optional[Json] = None) -> Json: """Get description of a given root.""" root = repos[name].get(root_name) if root is None: if default_root is not None: return default_root else: fail("Did not find mandatory root %s" % (name, )) if isinstance(root, str): return get_root(repos, root) return root def repo_roots_equal(repos: Json, name_a: str, name_b: str) -> bool: """Equality condition between repositories with respect to roots.""" if name_a == name_b: return True root_a = None root_b = None for root_name in REPO_ROOTS: root_a = get_root(repos, name_a, root_name=root_name, default_root=root_a) root_b = get_root(repos, name_b, root_name=root_name, default_root=root_b) if not roots_equal(root_a, root_b, name_a, name_b): return False for file_name, default_name in [("target_file_name", "TARGETS"), ("rule_file_name", "RULES"), ("expression_file_name", "EXPRESSIONS") ]: fname_a = repos[name_a].get(file_name, default_name) fname_b = repos[name_b].get(file_name, default_name) if fname_a != fname_b: return False return True names = sorted(repos.keys()) for j in range(len(names)): b = names[j] for i in range(j): a = names[i] if is_different(a, b): continue # Check equality between roots if not repo_roots_equal(repos, a, b): mark_as_different(a, b) continue # Check equality between bindings links_a = repos[a].get("bindings", {}) links_b = repos[b].get("bindings", {}) if set(links_a.keys()) != set(links_b.keys()): mark_as_different(a, b) continue for link in links_a.keys(): next_a = links_a[link] next_b = links_b[link] if next_a != next_b: if is_different(next_a, next_b): mark_as_different(a, b) continue else: # equality pending binding equality register_dependency(next_a, next_b, a, b) classes: List[List[str]] = [] done: Dict[str, bool] = {} for j in reversed(range(len(names))): name_j = names[j] if done.get(name_j): continue c = [name_j] for i in range(j): name_i = names[i] if not bisim.get((name_i, name_j), {}).get("different"): c.append(name_i) done[name_i] = True classes.append(c) return classes def deduplicate(repos: Json, user_keep: List[str]) -> Json: """Deduplicate repository names in a configuration object, keeping the main repository and names provided explicitly by the user.""" keep = set(user_keep) main = repos.get("main") if isinstance(main, str): keep.add(main) def choose_representative(c: List[str]) -> str: """Out of a bisimilarity class chose a main representative.""" candidates = c # Keep a repository with a proper root, if any of those has a root. # In this way, we're not losing actual roots. with_root = [ n for n in candidates if isinstance(repos["repositories"][n]["repository"], dict) ] if with_root: candidates = with_root # Prefer to choose a repository we have to keep anyway keep_entries = set(candidates) & keep if keep_entries: candidates = list(keep_entries) return sorted(candidates, key=lambda s: (s.count("/"), len(s), s))[0] def merge_pragma(rep: str, merged: List[str]) -> Json: """Merge the pragma fields in a sensible manner.""" desc = cast(Union[Any, Json], repos["repositories"][rep]["repository"]) if not isinstance(desc, dict): return desc pragma = desc.get("pragma", {}) # Clear pragma absent unless all merged repos that are not references # have the pragma absent: bool = pragma.get("absent", False) for c in merged: alt_desc = cast(Union[Any, Json], repos["repositories"][c]["repository"]) if (isinstance(alt_desc, dict)): absent = \ absent and alt_desc.get("pragma", {}).get("absent", False) pragma = dict(pragma, **{"absent": absent}) if not absent: del pragma["absent"] # Add pragma to_git if at least one of the merged repos requires it to_git = pragma.get("to_git", False) for c in merged: alt_desc = cast(Union[Any, Json], repos["repositories"][c]["repository"]) if (isinstance(alt_desc, dict)): to_git = \ to_git or alt_desc.get("pragma", {}).get("to_git", False) pragma = dict(pragma, **{"to_git": to_git}) if not to_git: del pragma["to_git"] # Update the pragma desc = dict(desc, **{"pragma": pragma}) if not pragma: del desc["pragma"] return desc bisim = bisimilar_repos(repos["repositories"]) renaming: Dict[str, str] = {} updated_repos: Json = {} for c in bisim: if len(c) == 1: continue rep = choose_representative(c) updated_repos[rep] = merge_pragma(rep, c) for repo in c: if ((repo not in keep) and (repo != rep)): renaming[repo] = rep def final_root_reference(name: str) -> str: """For a given repository name, return a name than can be used to name root in the final repository configuration.""" root: Any = repos["repositories"][name]["repository"] if isinstance(root, dict): # actual root; can still be merged into a different once, but only # one with a proper root as well. return renaming.get(name, name) if isinstance(root, str): return final_root_reference(root) fail("Invalid root found for %r: %r" % (name, root)) new_repos: Json = {} for name in repos["repositories"].keys(): if name not in renaming: desc: Json = repos["repositories"][name] if name in updated_repos: desc = dict(desc, **{"repository": updated_repos[name]}) if "bindings" in desc: bindings = desc["bindings"] new_bindings = {} for k, v in bindings.items(): if v in renaming: new_bindings[k] = renaming[v] else: new_bindings[k] = v desc = dict(desc, **{"bindings": new_bindings}) new_roots: Json = {} for root in ["repository", "target_root", "rule_root"]: root_val = desc.get(root) if isinstance(root_val, str) and (root_val in renaming): new_roots[root] = final_root_reference(root_val) desc = dict(desc, **new_roots) # Update target repos of precomputed roots if isinstance(desc.get("repository"), dict): repo_root: Json = desc["repository"] if repo_root["type"] in ["computed", "tree structure"] and \ repo_root["repo"] in renaming: repo_root = \ dict(repo_root, **{"repo": renaming[repo_root["repo"]]}) desc = dict(desc, **{"repository": repo_root}) new_repos[name] = desc return dict(repos, **{"repositories": new_repos}) ### # Core logic ## def lock_config(input_file: str) -> Json: """Main logic of just-lock.""" input_config: Json = {} try: with open(input_file) as f: input_config = json.load(f) except OSError: fail("Failed to open input file %s" % (input_file, )) except Exception as ex: fail("Reading input file %s failed with:\n%r" % (input_file, ex)) # Get the known fields main: Optional[str] = None if input_config.get("main") is not None: main = input_config.get("main") if not isinstance(main, str): fail("Expected field \"main\" to be a string, but found:\n%r" % (json.dumps(main, indent=2), )) repositories: Json = input_config.get("repositories", DEFAULT_REPO) if not repositories: # if empty, set it as default repositories = DEFAULT_REPO imports: List[Any] = input_config.get("imports", []) if not isinstance(imports, list): fail("Expected field \"imports\" to be a list, but found:\n%r" % (json.dumps(imports, indent=2), )) keep: List[str] = input_config.get("keep", []) if not isinstance(keep, list): fail("Expected field \"keep\" to be a list, but found:\n%r" % (json.dumps(keep, indent=2), )) # Acquire garbage collector locks git_gc_lock = gc_repo_lock_acquire(is_shared=True) storage_gc_lock = gc_storage_lock_acquire(is_shared=True) # Do checkouts asynchronously checkouts: Dict[int, Optional[CheckoutInfo]] = {} def run_checkout(*, source: str, key: int, entry: Json) -> None: """Run checkout and updates the outer variable 'checkouts'. Updates are atomic, so no extra locking is needed.""" if source == "git": checkouts[key] = git_checkout(entry) elif source == "archive": checkouts[key] = archive_checkout(entry) elif source == "git tree": checkouts[key] = git_tree_checkout(entry) report("Check out sources") with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as ts: for index, entry in enumerate(imports): if not isinstance(entry, dict): fail("Expected import entries to be objects, but found:\n%r" % (json.dumps(entry, indent=2), )) entry = cast(Json, entry) source = entry.get("source") if not isinstance(source, str): fail( "Expected field \"source\" to be a string, but found:\n%r" % (json.dumps(source, indent=2), )) # Add task only for sources that do work if source in ["git", "archive", "git tree"]: ts.submit(run_checkout, source=source, key=index, entry=entry) # Initialize the core config, which will be extended with imports core_config: Json = {} if main is not None: core_config["main"] = main core_config["repositories"] = repositories # Handle imports for index, entry in enumerate(imports): if not isinstance(entry, dict): fail("Expected import entries to be objects, but found:\n%r" % (json.dumps(entry, indent=2), )) entry = cast(Json, entry) source = entry.get("source") if not isinstance(source, str): fail("Expected field \"source\" to be a string, but found:\n%r" % (json.dumps(source, indent=2), )) if source == "git": # Get checkout info checkout_info = checkouts[index] if checkout_info is not None: core_config["repositories"] = import_from_git( core_config["repositories"], entry, checkout_info) elif source == "file": core_config["repositories"] = import_from_file( core_config["repositories"], entry) elif source == "archive": checkout_info = checkouts[index] if checkout_info is not None: core_config["repositories"] = import_from_archive( core_config["repositories"], entry, checkout_info) elif source == "git tree": checkout_info = checkouts[index] if checkout_info is not None: core_config["repositories"] = import_from_git_tree( core_config["repositories"], entry, checkout_info) elif source == "generic": core_config = import_generic(core_config, entry) else: fail("Unknown source for import entry \n%r" % (json.dumps(entry, indent=2), )) # Clone specified Git repositories locally asynchronously rewritten_repos: Json = {} def run_clone(*, repos: Json, clone_to: str, known_repo: str, deps_chain: List[str]) -> None: """Perform the cloning and update, if needed, the outer 'keep' list and fill in the 'rewritten_repos' dict. These updates are atomic, so no extra locking is needed.""" # Find target repository and clone its workspace root result = clone_repo(repos, known_repo, deps_chain, clone_to) if result is not None: target_repo, cloned_repo = result # Rewrite description of target repo to point to clone location rewritten_repos[target_repo] = rewrite_cloned_repo( repos, clone_to=clone_to, target_repo=target_repo, ws_root_repo=cloned_repo) # Add start and target repos to 'keep' list keep.extend([known_repo, target_repo]) if g_CLONE_MAP: report("Clone repositories") with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as ts: for clone_to, (known_repo, deps_chain) in g_CLONE_MAP.items(): ts.submit(run_clone, repos=core_config["repositories"], clone_to=clone_to, known_repo=known_repo, deps_chain=deps_chain) core_config["repositories"].update(rewritten_repos) # Release garbage collector locks lock_release(storage_gc_lock) lock_release(git_gc_lock) # Deduplicate output config core_config = deduplicate(core_config, keep) return core_config def main(): parser = ArgumentParser( prog="just-lock", formatter_class=RawTextHelpFormatter, description="Generate or update a multi-repository configuration file", exit_on_error=False, # catch parsing errors ourselves ) parser.add_argument("-C", dest="input_file", help="Input configuration file", metavar="FILE") parser.add_argument("-o", dest="output_file", help="Output multi-repository configuration file", metavar="FILE") parser.add_argument("--local-build-root", dest="local_build_root", help="Root for CAS, repository space, etc", metavar="PATH") parser.add_argument("--just", dest="just_bin", help="Path to the 'just' binary", metavar="PATH") parser.add_argument("--git", dest="git_bin", help="Git binary to use", metavar="PATH") parser.add_argument("--launcher", dest="launcher", help="Local launcher to use for commands in imports", metavar="JSON") parser.add_argument( "--clone", dest="clone", help="\n".join([ "Mapping from filesystem path to pair of repository name and list of bindings.", "Clone at path the workspace root of a repository found by following the bindings from named repository.", "IMPORTANT: The output configuration will point to the cloned repositories!" ]), metavar="JSON") try: args = parser.parse_args() except ArgumentError as err: fail("Failed parsing command line arguments with:\n%s" % (err.message, )) # Get the path of the input file if args.input_file: input_file = cast(str, args.input_file) # if explicitly provided, it must be an existing file if not os.path.isfile(input_file): fail("Provided input path '%s' is not a file!" % (input_file, )) else: # search usual paths in workspace root input_file = get_repository_config_file(DEFAULT_INPUT_CONFIG_NAME) if not input_file: fail("Failed to find an input file in the workspace") # Get the path of the output file if args.output_file: output_file = cast(str, args.output_file) else: # search usual paths in workspace root output_file = get_repository_config_file(DEFAULT_JUSTMR_CONFIG_NAME) if not output_file: # set it next to the input file parent_path = Path(input_file).parent output_file = os.path.realpath( os.path.join(parent_path, DEFAULT_JUSTMR_CONFIG_NAME)) # Process the rest of the command line; use globals for simplicity global g_ROOT, g_JUST, g_GIT, g_LAUNCHER, g_CLONE_MAP if args.local_build_root: g_ROOT = os.path.abspath(args.local_build_root) if args.just_bin: g_JUST = args.just_bin if args.git_bin: g_GIT = cast(str, args.git_bin) if args.launcher: g_LAUNCHER = json.loads(args.launcher) if args.clone: g_CLONE_MAP = json.loads(args.clone) out_config = lock_config(input_file) with open(output_file, "w") as f: json.dump(out_config, f, indent=2) if __name__ == "__main__": main() just-buildsystem-justbuild-b1fb5fa/bin/just-mr.py000077500000000000000000001254141516554100600223540ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import json import os import shutil import subprocess import sys import tempfile import time import zlib from typing import Any, Dict, List, NoReturn, Optional, Set, Tuple, Union, cast from argparse import ArgumentParser from pathlib import Path from enum import Enum # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters Json = Dict[str, Any] g_JUST: str = "just" g_JUST_ARGS: Dict[str, str] = {} g_ROOT: str = "/justroot" SYSTEM_ROOT: str = os.path.abspath(os.sep) g_WORKSPACE_ROOT: Optional[str] = None g_SETUP_ROOT: Optional[str] = None g_DISTDIR: List[str] = [] MARKERS: List[str] = [".git", "ROOT", "WORKSPACE"] g_ALWAYS_FILE: bool = False g_GIT_CHECKOUT_LOCATIONS_FILE: Optional[str] = None g_GIT_CHECKOUT_LOCATIONS: Dict[str, str] = {} TAKE_OVER: List[str] = [ "bindings", "target_file_name", "rule_file_name", "expression_file_name", ] ALT_DIRS: List[str] = [ "target_root", "rule_root", "expression_root", ] GIT_NOBODY_ENV: Dict[str, str] = { "GIT_AUTHOR_DATE": "1970-01-01T00:00Z", "GIT_AUTHOR_NAME": "Nobody", "GIT_AUTHOR_EMAIL": "nobody@example.org", "GIT_COMMITTER_DATE": "1970-01-01T00:00Z", "GIT_COMMITTER_NAME": "Nobody", "GIT_COMMITTER_EMAIL": "nobody@example.org", "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", } KNOWN_JUST_SUBCOMMANDS: Json = { "version": { "config": False, "build root": False }, "describe": { "config": True, "build root": False }, "analyse": { "config": True, "build root": True }, "build": { "config": True, "build root": True }, "install": { "config": True, "build root": True }, "rebuild": { "config": True, "build root": True }, "install-cas": { "config": False, "build root": True }, "gc": { "config": False, "build root": True } } g_CURRENT_SUBCOMMAND: Optional[str] = None LOCATION_TYPES: List[str] = ["workspace", "home", "system"] DEFAULT_RC_PATH: str = os.path.join(Path.home(), ".just-mrrc") DEFAULT_BUILD_ROOT: str = os.path.join(Path.home(), ".cache/just") DEFAULT_CHECKOUT_LOCATIONS_FILE: str = os.path.join(Path.home(), ".just-local.json") DEFAULT_DISTDIRS: List[str] = [os.path.join(Path.home(), ".distfiles")] DEFAULT_CONFIG_LOCATIONS: List[Json] = [{ "root": "workspace", "path": "repos.json" }, { "root": "workspace", "path": "etc/repos.json" }, { "root": "home", "path": ".just-repos.json" }, { "root": "system", "path": "etc/just-repos.json" }] class ObjectType(Enum): FILE = 1 EXEC = 2 LINK = 3 DIR = 4 def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def fail(s: str, exit_code: int = 65) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) def try_rmtree(tree: str) -> None: for _ in range(10): try: shutil.rmtree(tree) return except: time.sleep(1.0) fail("Failed to remove %s" % (tree, )) def move_to_place(src: str, dst: str) -> None: os.makedirs(os.path.dirname(dst), exist_ok=True) try: os.rename(src, dst) except Exception as e: if not os.path.exists(dst): fail("Failed to move %s to %s: %s" % (src, dst, e)) if not os.path.exists(dst): fail("%s not present after move" % (dst, )) def run_cmd(cmd: List[str], *, env: Any = None, stdout: Any = subprocess.DEVNULL, stdin: Any = None, cwd: str, attempts: int = 1) -> None: attempts = max(attempts, 1) # at least one attempt for _ in range(attempts): if subprocess.run(cmd, cwd=cwd, env=env, stdout=stdout, stdin=stdin).returncode == 0: return time.sleep(1.0) fail("Command %s in %s failed" % (cmd, cwd)) def find_workspace_root() -> Optional[str]: def is_workspace_root(path: str) -> bool: for m in MARKERS: if os.path.exists(os.path.join(path, m)): return True return False path: str = os.getcwd() while True: if is_workspace_root(path): return path if path == SYSTEM_ROOT: return None path = os.path.dirname(path) def read_config(configfile: str) -> Json: with open(configfile) as f: return json.load(f) def git_root(*, upstream: Optional[str]) -> str: if upstream is not None and upstream in g_GIT_CHECKOUT_LOCATIONS: return g_GIT_CHECKOUT_LOCATIONS[upstream] elif upstream and os.path.isabs(upstream) and os.path.isdir(upstream): return upstream else: return os.path.join(g_ROOT, "repositories/generation-0/git") def is_cache_git_root(upstream: Optional[str]) -> bool: """return true if upstream is the default git root in the cache directory""" return git_root(upstream=upstream) == git_root(upstream=None) def git_keep(commit: str, *, upstream: Optional[str]) -> None: if not is_cache_git_root(upstream): # for those, we assume the referenced commit is kept by # some branch anyway return git_env = {**os.environ, **GIT_NOBODY_ENV} run_cmd([ "git", "tag", "-f", "-m", "Keep referenced tree alive", "keep-%s" % (commit, ), commit ], cwd=git_root(upstream=upstream), env=git_env, attempts=3) def git_init_options(*, upstream: Optional[str]) -> List[str]: if upstream in g_GIT_CHECKOUT_LOCATIONS: return [] else: return ["--bare"] def ensure_git(*, upstream: Optional[str]) -> None: root: str = git_root(upstream=upstream) if os.path.exists(root): return os.makedirs(root) run_cmd(["git", "init"] + git_init_options(upstream=upstream), cwd=root) def git_commit_present(commit: str, *, upstream: str) -> bool: result = subprocess.run(["git", "show", "--oneline", commit], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=git_root(upstream=upstream)) return result.returncode == 0 def git_url_is_path(url: str) -> bool: for prefix in ["ssh://", "http://", "https://"]: if url.startswith(prefix): return False return True def git_fetch(*, repo: str, branch: str) -> None: if git_url_is_path(repo): repo = os.path.abspath(repo) run_cmd(["git", "fetch", repo, branch], cwd=git_root(upstream=repo)) def subdir_path(checkout: str, desc: Json) -> str: return os.path.normpath( os.path.join(checkout, cast(str, desc.get("subdir", ".")))) def git_tree(*, commit: str, subdir: str, upstream: str) -> str: tree = subprocess.run( ["git", "log", "-n", "1", "--format=%T", commit], stdout=subprocess.PIPE, cwd=git_root(upstream=upstream)).stdout.decode('utf-8').strip() return git_subtree(tree=tree, subdir=subdir, upstream=upstream) def git_subtree(*, tree: str, subdir: str, upstream: Optional[str]) -> str: if subdir == ".": return tree return subprocess.Popen( ["git", "cat-file", "--batch-check=%(objectname)"], stdout=subprocess.PIPE, stdin=subprocess.PIPE, cwd=git_root(upstream=upstream)).communicate( input=("%s:%s" % (tree, subdir)).encode())[0].decode('utf-8').strip() def git_checkout_dir(commit: str) -> str: return os.path.join(g_ROOT, "workspaces", "git", commit) def git_checkout(desc: Json) -> List[str]: commit: str = desc["commit"] target: str = git_checkout_dir(commit) if g_ALWAYS_FILE and os.path.exists(target): return ["file", subdir_path(target, desc)] repo: str = desc["repository"] if git_url_is_path(repo): repo = os.path.abspath(repo) root: str = git_root(upstream=repo) ensure_git(upstream=repo) if not git_commit_present(commit, upstream=repo): branch: str = desc["branch"] log("Fetching %s from %s (in %s)" % (branch, repo, root)) git_fetch(repo=repo, branch=branch) if not git_commit_present(commit, upstream=repo): fail("Fetching %s from %s failed to fetch %s" % (branch, repo, commit)) git_keep(commit, upstream=repo) if g_ALWAYS_FILE: if os.path.exists(target): try_rmtree(target) os.makedirs(target) with tempfile.TemporaryFile() as f: run_cmd(["git", "archive", commit], cwd=root, stdout=f) f.seek(0) run_cmd(["tar", "x"], cwd=target, stdin=f) return ["file", subdir_path(target, desc)] tree: str = git_tree(commit=commit, subdir=desc.get("subdir", "."), upstream=repo) return ["git tree", tree, root] def update_git(desc: Json) -> None: repo: str = desc["repository"] branch: str = desc["branch"] lsremote = subprocess.run(["git", "ls-remote", repo, branch], stdout=subprocess.PIPE).stdout desc["commit"] = lsremote.decode('utf-8').split('\t')[0] def git_hash(content: bytes, type: str = "blob") -> Tuple[str, bytes]: # Returns the git hash of the object, as well as the header to be stored header = "{} {}\0".format(type, len(content)).encode('utf-8') h = hashlib.sha1() h.update(header) h.update(content) return h.hexdigest(), header def add_to_cas(data: Union[str, bytes]) -> str: if isinstance(data, str): data = data.encode('utf-8') h, _ = git_hash(data) cas_root = os.path.join( g_ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}") basename = h[2:] target = os.path.join(cas_root, basename) tempname = os.path.join(cas_root, "%s.%d" % (basename, os.getpid())) if os.path.exists(target): return target os.makedirs(cas_root, exist_ok=True) with open(tempname, "wb") as f: f.write(data) f.flush() os.chmod(f.fileno(), 0o444) os.fsync(f.fileno()) os.utime(tempname, (0, 0)) os.rename(tempname, target) return target def cas_path(h: str) -> str: return os.path.join( g_ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}", h[2:]) def is_in_cas(h: str) -> bool: return os.path.exists(cas_path(h)) def add_file_to_cas(filename: str) -> None: # TODO: avoid going through memory with open(filename, "rb") as f: data = f.read() add_to_cas(data) def add_distfile_to_cas(distfile: str) -> None: for d in g_DISTDIR: candidate = os.path.join(d, distfile) if os.path.exists(candidate): add_file_to_cas(candidate) def archive_checkout_dir(content: str, repo_type: str) -> str: return os.path.join(g_ROOT, "workspaces", repo_type, content) def archive_tmp_checkout_dir(content: str, repo_type: str) -> str: return os.path.join(g_ROOT, "tmp-workspaces", repo_type, "%d-%s" % (os.getpid(), content)) def archive_tree_id_file(content: str, repo_type: str) -> str: return os.path.join(g_ROOT, "repositories/generation-0/tree-map", repo_type, content) def get_distfile(desc: Json) -> str: distfile = desc.get("distfile") if not distfile: distfile = os.path.basename(cast(str, desc.get("fetch"))) return distfile def archive_fetch(desc: Json, content: str) -> None: """ Makes sure archive is available and accounted for in cas """ if not is_in_cas(content): distfile = get_distfile(desc) if distfile: add_distfile_to_cas(distfile) if not is_in_cas(content): url: str = desc["fetch"] data = subprocess.run(["wget", "-O", "-", url], stdout=subprocess.PIPE).stdout if "sha256" in desc: actual_hash = hashlib.sha256(data).hexdigest() if desc["sha256"] != actual_hash: fail("SHA256 mismatch for %s, expected %s, found %s" % (url, desc["sha256"], actual_hash)) if "sha512" in desc: actual_hash = hashlib.sha512(data).hexdigest() if desc["sha512"] != actual_hash: fail("SHA512 mismatch for %s, expected %s, found %s" % (url, desc["sha512"], actual_hash)) add_to_cas(data) if not is_in_cas(content): fail("Failed to fetch a file with id %s from %s" % (content, url)) def archive_checkout(desc: Json, repo_type: str = "archive") -> List[str]: content_id: str = desc["content"] target: str = archive_checkout_dir(content_id, repo_type=repo_type) if g_ALWAYS_FILE and os.path.exists(target): return ["file", subdir_path(target, desc)] tree_id_file: str = archive_tree_id_file(content_id, repo_type=repo_type) if (not g_ALWAYS_FILE) and os.path.exists(tree_id_file): with open(tree_id_file) as f: archive_tree_id = f.read() # ensure git cache exists ensure_git(upstream=None) return [ "git tree", git_subtree(tree=archive_tree_id, subdir=desc.get("subdir", "."), upstream=None), git_root(upstream=None), ] archive_fetch(desc, content=content_id) target_tmp: str = archive_tmp_checkout_dir(content_id, repo_type=repo_type) if os.path.exists(target_tmp): try_rmtree(target_tmp) os.makedirs(target_tmp) if repo_type == "zip": try: run_cmd(["unzip", "-d", ".", cas_path(content_id)], cwd=target_tmp) except: try: run_cmd(["7z", "x", cas_path(content_id)], cwd=target_tmp) except: print("Failed to extract zip-like archive %s" % (cas_path(content_id), )) sys.exit(1) else: try: run_cmd(["tar", "xf", cas_path(content_id)], cwd=target_tmp) except: print("Failed to extract tarball %s" % (cas_path(content_id), )) sys.exit(1) if g_ALWAYS_FILE: move_to_place(target_tmp, target) return ["file", subdir_path(target, desc)] tree: str = import_to_git(target_tmp, repo_type, content_id) try_rmtree(target_tmp) os.makedirs(os.path.dirname(tree_id_file), exist_ok=True) with open(tree_id_file, "w") as f: f.write(tree) return [ "git tree", git_subtree(tree=tree, subdir=desc.get("subdir", "."), upstream=None), git_root(upstream=None) ] def import_tmp_dir(content: str) -> str: return os.path.join(g_ROOT, "tmp-workspaces", "import", "%d-%s" % (os.getpid(), content)) def type_to_perm(obj_type: ObjectType) -> str: if obj_type == ObjectType.DIR: return "40000" elif obj_type == ObjectType.LINK: return "120000" elif obj_type == ObjectType.EXEC: return "100755" else: # obj_type == ObjectType.FILE return "100644" def write_blob_to_repo(repo_root: str, data: bytes) -> bytes: # Get blob hash and header to be stored h, header = git_hash(data, type="blob") # Write repository object obj_dir = "{}/.git/objects/{}".format(repo_root, h[0:2]) obj_file = "{}/{}".format(obj_dir, h[2:]) os.makedirs(obj_dir, exist_ok=True) with open(obj_file, "wb") as f: f.write(zlib.compress(header + data)) return bytes.fromhex(h) # raw id def write_tree_to_repo(repo_root: str, entries: Dict[str, Tuple[bytes, ObjectType]]) -> bytes: # Tree entries have as key their filename and as value a tuple of raw id and # object type. They must be sorted by filename. tree_content: bytes = b"" for fname, entry in sorted(entries.items()): if entry[1] == ObjectType.DIR: fname = fname[:-1] # remove trailing '/' tree_content += "{} {}\0".format(type_to_perm(entry[1]), fname).encode('utf-8') + entry[0] # Get tree hash and header to be stored h, header = git_hash(tree_content, type="tree") # Write repository object obj_dir = "{}/.git/objects/{}".format(repo_root, h[0:2]) obj_file = "{}/{}".format(obj_dir, h[2:]) os.makedirs(obj_dir, exist_ok=True) with open(obj_file, "wb") as f: f.write(zlib.compress(header + tree_content)) return bytes.fromhex(h) # raw id def path_to_type(fpath: str) -> ObjectType: if os.path.islink(fpath): return ObjectType.LINK elif os.path.isdir(fpath): return ObjectType.DIR else: if os.access(fpath, os.X_OK): return ObjectType.EXEC else: return ObjectType.FILE def get_tree_raw_id(source_dir: str, repo_root: str) -> bytes: # Writes the content of the directory recursively to the repository and # returns its sha1 hash and its raw bytes representation entries: Dict[str, Tuple[bytes, ObjectType]] = {} for fname in os.listdir(source_dir): fpath = source_dir + "/" + fname obj_type = path_to_type(fpath) raw_h: bytes = b"" if obj_type == ObjectType.DIR: raw_h = get_tree_raw_id(fpath, repo_root) fname = fname + '/' # trailing '/' added for correct sorting elif obj_type == ObjectType.LINK: data = os.readlink(fpath).encode('utf-8') raw_h = write_blob_to_repo(repo_root, data) else: with open(fpath, "rb") as f: data = f.read() raw_h = write_blob_to_repo(repo_root, data) # Add entry to map entries[fname] = (raw_h, obj_type) return write_tree_to_repo(repo_root, entries) def import_to_git(target: str, repo_type: str, content_id: str) -> str: # In order to import content that might otherwise be ignored by Git, such # as empty directories or magic-named files and folders (e.g., .git, # .gitignore), add entries manually to the repository, which should be in # its own separate location repo_tmp_dir = import_tmp_dir(content_id) if os.path.exists(repo_tmp_dir): try_rmtree(repo_tmp_dir) os.makedirs(repo_tmp_dir) # Initialize repo to have access to its storage git_env = {**os.environ, **GIT_NOBODY_ENV} run_cmd( ["git", "init"], cwd=repo_tmp_dir, env=git_env, ) # Get tree id of added directory tree_id: str = get_tree_raw_id(target, repo_tmp_dir).hex() # Commit the tree commit: str = subprocess.run( [ "git", "commit-tree", tree_id, "-m", "Content of %s %r" % (repo_type, content_id) ], stdout=subprocess.PIPE, cwd=repo_tmp_dir, env=git_env, ).stdout.decode('utf-8').strip() # Update the HEAD to make the tree fetchable run_cmd( ["git", "update-ref", "HEAD", commit], cwd=repo_tmp_dir, env=git_env, ) # Fetch commit into Git cache repository and tag it ensure_git(upstream=None) run_cmd(["git", "fetch", repo_tmp_dir], cwd=git_root(upstream=None)) git_keep(commit, upstream=None) return tree_id def file_as_git(fpath: str) -> List[str]: root_result = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=fpath, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) if not root_result.returncode == 0: # TODO: consider also doing this for pending changes # copy non-git paths to tmp-workspace and import to git fpath = os.path.realpath(fpath) if g_CURRENT_SUBCOMMAND: log(f"""\ Warning: Inefficient Git import of file path '{fpath}'. Please consider using 'just-mr setup' and 'just {g_CURRENT_SUBCOMMAND}' separately to cache the output.""") target = archive_tmp_checkout_dir(os.path.relpath(fpath, "/"), repo_type="file") os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copytree(fpath, target) tree = import_to_git(target, "file", fpath) try_rmtree(target) return ["git tree", tree, git_root(upstream=None)] root = root_result.stdout.decode('utf-8').rstrip() subdir = os.path.relpath(fpath, root) root_tree = subprocess.run( ["git", "log", "-n", "1", "--format=%T"], cwd=root, stdout=subprocess.PIPE).stdout.decode('utf-8').strip() return [ "git tree", git_subtree(tree=root_tree, subdir=subdir, upstream=root), root ] def file_checkout(desc: Json) -> List[str]: fpath = os.path.abspath(desc["path"]) if desc.get("pragma", {}).get("to_git") and not g_ALWAYS_FILE: return file_as_git(fpath) return ["file", os.path.abspath(fpath)] # strongly imposing typing on this method is complicated, so resort to Any def resolve_repo(desc: Any, *, seen: Optional[List[str]] = None, repos: Json) -> Json: seen = seen or [] if not isinstance(desc, str): return desc if desc in seen: fail("Cyclic reference in repository source definition: %r" % (seen, )) return resolve_repo(repos[desc]["repository"], seen=seen + [desc], repos=repos) def distdir_repo_dir(content: str) -> str: return os.path.join(g_ROOT, "distdir", content) def distdir_tmp_dir(content: str) -> str: return os.path.join(g_ROOT, "tmp-workspaces", "distdir", "%d-%s" % (os.getpid(), content)) def distdir_tree_id_file(content: str) -> str: return os.path.join(g_ROOT, "repositories/generation-0/distfiles-tree-map", content) def distdir_checkout(desc: Json, repos: Json) -> List[str]: """ Logic for processing the distdir repo type. """ content: Dict[str, str] = {} # All repos in distdir list are considered, irrespective of reachability distdir_repos: List[str] = desc.get("repositories", []) for repo in distdir_repos: # If repo does not exist, fail if repo not in repos: print("No configuration for repository %s found" % (repo, )) sys.exit(1) repo_desc: Json = repos[repo].get("repository", {}) repo_desc_type = repo_desc.get("type") # Only do work for archived types if repo_desc_type in ["archive", "zip"]: # fetch repo content_id: str = repo_desc["content"] # Store the relevant info in the map content[get_distfile(repo_desc)] = content_id # Hash the map as unique id for the distdir repo entry distdir_content_id, _ = git_hash( json.dumps(content, sort_keys=True, separators=(',', ':')).encode('utf-8')) target_distdir_dir = distdir_repo_dir(distdir_content_id) # Check if content already exists if g_ALWAYS_FILE and os.path.exists(target_distdir_dir): return ["file", target_distdir_dir] tree_id_file = distdir_tree_id_file(distdir_content_id) if (not g_ALWAYS_FILE) and os.path.exists(tree_id_file): with open(tree_id_file) as f: distdir_tree_id = f.read() # ensure git cache exists ensure_git(upstream=None) return [ "git tree", git_subtree(tree=distdir_tree_id, subdir=".", upstream=None), git_root(upstream=None) ] # As the content is not there already, so we have to ensure the archives # are present. for repo in distdir_repos: repo_desc = repos[repo].get("repository", {}) repo_desc_type = repo_desc.get("type") if repo_desc_type in ["archive", "zip"]: content_id = repo_desc["content"] archive_fetch(repo_desc, content=content_id) # Create the dirstdir repo folder content target_tmp_dir = distdir_tmp_dir(distdir_content_id) if os.path.exists(target_tmp_dir): try_rmtree(target_tmp_dir) os.makedirs(target_tmp_dir) for name, content_id in content.items(): target = os.path.join(target_tmp_dir, name) os.link(cas_path(content_id), target) if g_ALWAYS_FILE: # Return with path to distdir repo move_to_place(target_tmp_dir, target_distdir_dir) return ["file", target_distdir_dir] # Gitify the distdir repo folder tree = import_to_git(target_tmp_dir, "distdir", distdir_content_id) try_rmtree(target_tmp_dir) # Cache git info to tree id file os.makedirs(os.path.dirname(tree_id_file), exist_ok=True) with open(tree_id_file, "w") as f: f.write(tree) # Return git tree info return [ "git tree", git_subtree(tree=tree, subdir=".", upstream=None), git_root(upstream=None) ] def checkout(desc: Json, *, name: str, repos: Json) -> List[str]: repo_desc = resolve_repo(desc, repos=repos) repo_type = repo_desc.get("type") if repo_type == "git": return git_checkout(repo_desc) if repo_type in ["archive", "zip"]: return archive_checkout(repo_desc, repo_type=cast(str, repo_type)) if repo_type == "file": return file_checkout(repo_desc) if repo_type == "distdir": return distdir_checkout(repo_desc, repos=repos) fail("Unknown repository type %s for %s" % (repo_type, name)) def reachable_repositories(repo: str, *, repos: Json) -> Tuple[Set[str], Set[str]]: # First compute the set of repositories transitively reachable via bindings reachable: Set[str] = set() def traverse(x: str) -> None: nonlocal reachable if x in reachable: return reachable.add(x) bindings = repos[x].get("bindings", {}) for bound in bindings.values(): traverse(bound) traverse(repo) # Now add the repositories that serve as overlay directories for # targets, rules, etc. Those repositories have to be fetched as well, but # we do not have to consider their bindings. to_fetch = reachable.copy() for x in reachable: for layer in ALT_DIRS: if layer in repos[x]: to_fetch.add(repos[x][layer]) return reachable, to_fetch def setup(*, config: Json, main: Optional[str], setup_all: bool = False, interactive: bool = False) -> str: cwd: str = os.getcwd() if g_SETUP_ROOT: os.chdir(g_SETUP_ROOT) repos: Json = config.get("repositories", {}) repos_to_setup = repos.keys() repos_to_include = repos.keys() mr_config: Json = {} if main is None: main = config.setdefault("main", None) if not isinstance(main, str) and main is not None: fail("Unsupported value for field 'main' in configuration.") if main is not None: # pass on main that was explicitly set via command line or config mr_config["main"] = main if main is None and len(repos_to_setup) > 0: main = sorted(list(repos_to_setup))[0] if main is not None and not setup_all: repos_to_include, repos_to_setup = reachable_repositories(main, repos=repos) mr_repos: Json = {} for repo in repos_to_setup: desc = repos[repo] if repo == main and interactive: config = {} else: workspace = checkout(desc.get("repository", {}), name=repo, repos=repos) config = {"workspace_root": workspace} for key in TAKE_OVER: val = desc.get(key, {}) if val: config[key] = val mr_repos[repo] = config # Alternate directories are specifies as the workspace of # some other repository. So we have to iterate over all repositories again # to add those directories. We do this only for the repositories we include # in the final configuration. for repo in repos_to_include: desc = repos[repo] if repo == main and interactive: continue for key in ALT_DIRS: val = desc.get(key, {}) if val: if val == main and interactive: continue mr_repos[repo][key] = mr_repos[val]["workspace_root"] mr_repos_actual: Json = {} for repo in repos_to_include: mr_repos_actual[repo] = mr_repos[repo] mr_config["repositories"] = mr_repos_actual if g_SETUP_ROOT: os.chdir(cwd) return add_to_cas(json.dumps(mr_config, indent=2, sort_keys=True)) def call_just(*, config: Json, main: Optional[str], args: List[str]) -> None: subcommand = args[0] if len(args) > 0 else None args = args[1:] use_config: bool = False use_build_root: bool = False setup_config = config # default to given config path if subcommand and subcommand in KNOWN_JUST_SUBCOMMANDS: if KNOWN_JUST_SUBCOMMANDS[subcommand]["config"]: use_config = True global g_CURRENT_SUBCOMMAND g_CURRENT_SUBCOMMAND = subcommand setup_config = setup(config=config, main=main) g_CURRENT_SUBCOMMAND = None use_build_root = KNOWN_JUST_SUBCOMMANDS[subcommand]["build root"] if not setup_config: fail("Setup failed") cmd: List[str] = [g_JUST] cmd += [subcommand] if subcommand else [] cmd += ["-C", cast(str, setup_config)] if use_config else [] cmd += ["--local-build-root", g_ROOT] if use_build_root else [] if subcommand and subcommand in g_JUST_ARGS: cmd += g_JUST_ARGS[subcommand] cmd += args log("Setup finished, exec %s" % (cmd, )) try: os.execvp(g_JUST, cmd) except Exception as e: log(str(e)) finally: fail(f"exec failed", 64) def update(*, config: Json, repos: Json) -> None: for repo in repos: desc: Json = config["repositories"][repo]["repository"] desc = resolve_repo(desc, repos=config["repositories"]) repo_type = desc.get("type") if repo_type == "git": update_git(desc) else: fail("Don't know how to update %s repositories" % (repo_type, )) print(json.dumps(config, indent=2)) sys.exit(0) def fetch(*, config: Json, fetch_dir: Optional[str], main: Optional[str], fetch_all: bool = False) -> None: if not fetch_dir: for d in g_DISTDIR: if os.path.isdir(d): fetch_dir = os.path.abspath(d) break if not fetch_dir: fail("No directory found to fetch to, considered %r" % (g_DISTDIR, )) repos = config["repositories"] repos_to_fetch = repos.keys() if not fetch_all: if main is None and len(repos_to_fetch) > 0: main = sorted(list(repos_to_fetch))[0] if main is not None: repos_to_fetch = reachable_repositories(main, repos=repos)[0] def is_subpath(path: str, base: str) -> bool: return os.path.commonpath([path, base]) == base # warn if fetch_dir is in invocation workspace if g_WORKSPACE_ROOT and is_subpath(fetch_dir, g_WORKSPACE_ROOT): repo = repos.get(main, {}).get("repository", {}) repo_path = repo.get("path", None) if repo_path is not None and repo.get("type", None) == "file": if not os.path.isabs(repo_path): repo_path = os.path.realpath( os.path.join(cast(str, g_SETUP_ROOT), repo_path)) # only warn if repo workspace differs to invocation workspace if not is_subpath(repo_path, g_WORKSPACE_ROOT): log(f"""\ Warning: Writing distribution files to workspace location '{fetch_dir}', which is different to the workspace of the requested main repository '{repo_path}'.""") print("Fetching to %r" % (fetch_dir, )) for repo in repos_to_fetch: desc = repos[repo] if ("repository" in desc and isinstance(desc["repository"], dict) and desc["repository"]["type"] in ["zip", "archive"]): repo_desc = desc["repository"] distfile = repo_desc.get("distfile") or os.path.basename( repo_desc["fetch"]) content = repo_desc["content"] print("%r --> %r (content: %s)" % (repo, distfile, content)) archive_fetch(repo_desc, content) shutil.copyfile(cas_path(content), os.path.join(fetch_dir, distfile)) sys.exit(0) def read_location(location: Json) -> Optional[Tuple[str, str]]: root = location.setdefault("root", None) path = location.setdefault("path", None) base = location.setdefault("base", ".") if not path or root not in LOCATION_TYPES: fail(f"malformed location object: {location}") if root == "workspace": if not g_WORKSPACE_ROOT: log(f"Warning: Not in workspace root, ignoring location %s." % (location, )) return None root = g_WORKSPACE_ROOT if root == "home": root = Path.home() if root == "system": root = SYSTEM_ROOT return os.path.realpath(os.path.join(cast(str, root), cast( str, path))), os.path.realpath(os.path.join(cast(str, root), base)) # read settings from just-mrrc and return config path def read_justmrrc(rcpath: str, no_rc: bool = False) -> Optional[str]: rc: Json = {} if not no_rc: if not rcpath: rcpath = DEFAULT_RC_PATH elif not os.path.isfile(rcpath): fail(f"cannot read rc file {rcpath}.") if os.path.isfile(rcpath): with open(rcpath) as f: rc = json.load(f) location: Optional[Json] = rc.get("local build root", None) build_root = read_location(location) if location else None if build_root: global g_ROOT g_ROOT = build_root[0] location = rc.get("checkout locations", None) checkout = read_location(location) if location else None if checkout: global g_GIT_CHECKOUT_LOCATIONS_FILE if not os.path.isfile(checkout[0]): fail(f"cannot find checkout locations files {checkout[0]}.") g_GIT_CHECKOUT_LOCATIONS_FILE = checkout[0] location = rc.get("distdirs", None) if location: global g_DISTDIR g_DISTDIR = [] for l in location: paths = read_location(cast(Json, l)) if paths: if os.path.isdir(paths[0]): g_DISTDIR.append(paths[0]) else: log(f"Warning: Ignoring non-existing distdir {paths[0]}.") location = rc.get("just", None) just = read_location(location) if location else None if just: global g_JUST g_JUST = just[0] global g_JUST_ARGS g_JUST_ARGS = rc.get("just args", {}) for location in rc.get("config lookup order", DEFAULT_CONFIG_LOCATIONS): paths = read_location(cast(Json, location)) if paths and os.path.isfile(paths[0]): global g_SETUP_ROOT g_SETUP_ROOT = paths[1] return paths[0] return None def main(): parser = ArgumentParser() parser.add_argument("-C", dest="repository_config", help="Repository-description file to use", metavar="FILE") parser.add_argument("--checkout-locations", dest="checkout_location", help="Specification file for checkout locations") parser.add_argument("--local-build-root", dest="local_build_root", help="Root for CAS, repository space, etc", metavar="PATH") parser.add_argument("--distdir", dest="distdir", action="append", default=[], help="Directory to look for distfiles before fetching", metavar="PATH") parser.add_argument("--just", dest="just", help="Path to the just binary", metavar="PATH") parser.add_argument("--always-file", dest="always_file", action="store_true", default=False, help="Always create file roots") parser.add_argument( "--main", dest="main", default=None, help="Main repository to consider from the configuration.") parser.add_argument("--rc", dest="rcfile", help="Use just-mrrc file from custom path.") parser.add_argument("--norc", dest="norc", action="store_true", default=False, help="Do not use any just-mrrc file.") subcommands = parser.add_subparsers(dest="subcommand", title="subcommands", required=True) repo_parser = ArgumentParser(add_help=False) repo_parser.add_argument( "--all", dest="sub_all", action="store_true", default=False, help="Consider all repositories in the configuration.") repo_parser.add_argument( "sub_main", metavar="main-repo", nargs="?", default=None, help="Main repository to consider from the configuration.") subcommands.add_parser("setup", parents=[repo_parser], help="Setup and generate just configuration.") subcommands.add_parser( "setup-env", parents=[repo_parser], help="Setup without workspace root for the main repository.") fetch_parser = subcommands.add_parser( "fetch", parents=[repo_parser], help="Fetch and store distribution files.") fetch_parser.add_argument("-o", dest="fetch_dir", help="Directory to write distfiles when fetching", metavar="PATH") update_parser = subcommands.add_parser( "update", help="Advance Git commit IDs and print updated just-mr configuration.") update_parser.add_argument("update_repos", metavar="repo", nargs="*", default=[], help="Repository to update.") subcommands.add_parser("do", help="Canonical way of specifying just subcommands", add_help=False) for cmd in KNOWN_JUST_SUBCOMMANDS: subcommands.add_parser(cmd, help=f"Run setup and call 'just {cmd}'", add_help=False) (options, args) = parser.parse_known_args() global g_ROOT, g_GIT_CHECKOUT_LOCATIONS_FILE, g_DISTDIR, g_SETUP_ROOT, g_WORKSPACE_ROOT g_ROOT = DEFAULT_BUILD_ROOT if os.path.isfile(DEFAULT_CHECKOUT_LOCATIONS_FILE): g_GIT_CHECKOUT_LOCATIONS_FILE = DEFAULT_CHECKOUT_LOCATIONS_FILE g_DISTDIR = DEFAULT_DISTDIRS g_SETUP_ROOT = os.path.curdir g_WORKSPACE_ROOT = find_workspace_root() config_file = read_justmrrc(options.rcfile, options.norc) if options.repository_config: config_file = options.repository_config if not config_file: fail("cannot find repository configuration.") config = read_config(cast(str, config_file)) if options.local_build_root: g_ROOT = os.path.abspath(options.local_build_root) if options.checkout_location: if not os.path.isfile(options.checkout_location): fail("cannot find checkout locations file %s" % (options.checkout_location, )) g_GIT_CHECKOUT_LOCATIONS_FILE = os.path.abspath( options.checkout_location) if g_GIT_CHECKOUT_LOCATIONS_FILE: with open(g_GIT_CHECKOUT_LOCATIONS_FILE) as f: global g_GIT_CHECKOUT_LOCATIONS g_GIT_CHECKOUT_LOCATIONS = json.load(f).get("checkouts", {}).get("git", {}) if options.distdir: g_DISTDIR += options.distdir global g_JUST if options.just: g_JUST = options.just global g_ALWAYS_FILE g_ALWAYS_FILE = options.always_file sub_main = options.sub_main if hasattr(options, "sub_main") else None sub_all = options.sub_all if hasattr(options, "sub_all") else None if not sub_all and options.main and sub_main and options.main != sub_main: print( "Warning: conflicting options for main repository, selecting '%s'." % (sub_main, )) main: Optional[str] = sub_main or options.main if options.subcommand in KNOWN_JUST_SUBCOMMANDS: call_just(config=config, main=main, args=[options.subcommand] + args) return if options.subcommand == "do": call_just(config=config, main=main, args=args) return if args: print("Warning: ignoring unknown arguments %r" % (args, )) if options.subcommand == "setup-env": # Setup for interactive use, i.e., fetch the required repositories # and generate an appropriate multi-repository configuration file. # Store it in the CAS and print its path on stdout. # # For the main repository (if specified), leave out the workspace # so that in the usage of just the workspace is determined from # the working directory; in this way, working on a checkout of that # repository is possible, while having all dependencies set up # correctly. print( setup(config=config, main=main, setup_all=options.sub_all, interactive=True)) return if options.subcommand == "setup": # Setup such that the main repository keeps its workspace given in # the input config. print(setup(config=config, main=main, setup_all=options.sub_all)) return if options.subcommand == "update": update(config=config, repos=options.update_repos) if options.subcommand == "fetch": fetch(config=config, fetch_dir=options.fetch_dir, main=main, fetch_all=options.sub_all) fail("Unknown subcommand %s" % (options.subcommand, )) if __name__ == "__main__": main() just-buildsystem-justbuild-b1fb5fa/bin/parallel-bootstrap-traverser.py000077500000000000000000000356071516554100600266010ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import json import multiprocessing import os import shutil import subprocess import sys import threading from enum import Enum from argparse import ArgumentParser from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, cast # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters Json = Dict[str, Any] class AtomicInt: # types of attributes __value: int __cv: threading.Condition def __init__(self, init: int = 0) -> None: self.__value = init self.__cv = threading.Condition() def __notify(self) -> None: """Must be called with acquired cv.""" if self.__value == 0: self.__cv.notify_all() @property def value(self) -> int: with self.__cv: return self.__value @value.setter def value(self, to: int) -> None: with self.__cv: self.__value = to self.__notify() def fetch_inc(self, by: int = 1) -> int: """Post-increment""" with self.__cv: val = self.__value self.__value += by self.__notify() return val def fetch_dec(self, by: int = 1) -> int: """Post-decrement""" return self.fetch_inc(-by) def wait_for_zero(self) -> None: with self.__cv: self.__cv.wait_for(lambda: self.__value == 0) class TaskSystem: """Simple queue-based task system.""" Task = Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]] class Queue: cv: threading.Condition def __init__(self) -> None: self.cv = threading.Condition() self.tasks: List[TaskSystem.Task] = [] # types of attributes __shutdown: bool __num_workers: int __current_idx: AtomicInt __qs: List[Queue] __total_work: AtomicInt __workers: List[threading.Thread] def __init__(self, max_workers: int = multiprocessing.cpu_count()) -> None: """Creates the task system with `max_workers` many threads.""" self.__shutdown = False self.__num_workers = max(1, max_workers) self.__current_idx = AtomicInt() self.__qs = [TaskSystem.Queue() for _ in range(self.__num_workers)] def run(q: TaskSystem.Queue, idx: int): try: while not self.__shutdown: task: Optional[TaskSystem.Task] = None with q.cv: if len(q.tasks) == 0: self.__total_work.fetch_dec() # suspend thread q.cv.wait_for( lambda: len(q.tasks) > 0 or self.__shutdown) self.__total_work.fetch_inc() # active thread if len(q.tasks) > 0 and not self.__shutdown: task = q.tasks.pop(0) self.__total_work.fetch_dec() if task: task[0](*task[1], **task[2]) except Exception as e: self.__shutdown = True raise e finally: self.__total_work.value = 0 # total work = num(queued tasks) + num(active threads) self.__total_work = AtomicInt(self.__num_workers) # initially active self.__workers = [ threading.Thread(target=run, args=(self.__qs[i], i)) for i in range(self.__num_workers) ] for w in self.__workers: w.start() def __enter__(self): return self def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: try: self.finish() finally: self.shutdown() def add(self, fn: Callable[..., None], *args: Any, **kw: Any) -> None: """Add task to task queue, might block.""" if not self.__shutdown: q = self.__qs[self.__current_idx.fetch_inc() % self.__num_workers] with q.cv: self.__total_work.fetch_inc() q.tasks.append((fn, args, kw)) q.cv.notify_all() def finish(self) -> None: """Wait for queued tasks and active threads to become zero.""" self.__total_work.wait_for_zero() def shutdown(self) -> None: """Initiate shutdown of task system and wait for all threads to stop.""" self.__shutdown = True # signal shutdown for q in self.__qs: # notify everyone about shutdown with q.cv: q.cv.notify_all() for w in self.__workers: # wait for workers to shutdown w.join() class AtomicListMap: class Entry(Enum): CREATED = 0 INSERTED = 1 CLEARED = 2 # types of attributes __map: Dict[str, Optional[List[Any]]] __lock: threading.Lock def __init__(self) -> None: self.__map = dict() self.__lock = threading.Lock() def add(self, key: str, val: Any): with self.__lock: if key not in self.__map: self.__map[key] = [val] return AtomicListMap.Entry.CREATED elif self.__map[key] != None: cast(List[Any], self.__map[key]).append(val) return AtomicListMap.Entry.INSERTED else: return AtomicListMap.Entry.CLEARED def fetch_clear(self, key: str) -> Optional[List[Any]]: with self.__lock: vals = self.__map[key] self.__map[key] = None return vals g_CALLBACKS_PER_ID = AtomicListMap() def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) def fail(s: str) -> NoReturn: log(s) sys.exit(1) def git_hash(content: bytes) -> str: header = "blob {}\0".format(len(content)).encode('utf-8') h = hashlib.sha1() h.update(header) h.update(content) return h.hexdigest() def create_blobs(blobs: List[str], *, root: str, ts: TaskSystem) -> None: os.makedirs(os.path.join(root, "KNOWN")) def write_blob(blob_bin: bytes) -> None: with open(os.path.join(root, "KNOWN", git_hash(blob_bin)), "wb") as f: f.write(blob_bin) for blob in blobs: ts.add(write_blob, blob.encode('utf-8')) def build_known(desc: Json, *, root: str) -> str: return os.path.join(root, "KNOWN", desc["data"]["id"]) def link(src: str, dest: str) -> None: dest = os.path.normpath(dest) os.makedirs(os.path.dirname(dest), exist_ok=True) try: os.link(src, dest) except: os.symlink(src, dest) def build_local(desc: Json, *, root: str, config: Json) -> str: repo_name: str = desc["data"]["repository"] repo: List[str] = config["repositories"][repo_name]["workspace_root"] rel_path: str = desc["data"]["path"] if repo[0] == "file": return os.path.join(repo[1], rel_path) fail("Unsupported repository root %r" % (repo, )) def build_tree(desc: Json, *, config: Json, root: str, graph: Json, ts: TaskSystem, callback: Callable[..., None]): tree_id = desc["data"]["id"] tree_dir = os.path.normpath(os.path.join(root, "TREE", tree_id)) state: AtomicListMap.Entry = g_CALLBACKS_PER_ID.add(f"TREE/{tree_id}", callback) if state != AtomicListMap.Entry.CREATED: # we are not first if state != AtomicListMap.Entry.INSERTED: # tree ready, run callback callback(tree_dir) return tree_dir_tmp = tree_dir + ".tmp" tree_desc: Json = graph["trees"][tree_id] num_entries = AtomicInt(len(tree_desc.items())) def run_callbacks() -> None: if num_entries.fetch_dec() <= 1: # correctly handle the empty tree os.makedirs(tree_dir_tmp, exist_ok=True) shutil.copytree(tree_dir_tmp, tree_dir) vals = g_CALLBACKS_PER_ID.fetch_clear(f"TREE/{tree_id}") if vals: for cb in vals: # mark ready ts.add(cb, tree_dir) if num_entries.value == 0: run_callbacks() for location, desc in tree_desc.items(): def create_link(location: str) -> Callable[..., None]: def do_link(path: str) -> None: link(path, os.path.join(tree_dir_tmp, location)) run_callbacks() return do_link ts.add(build, desc, config=config, root=root, graph=graph, ts=ts, callback=create_link(location)) def run_action(action_id: str, *, config: Json, root: str, graph: Json, ts: TaskSystem, callback: Callable[..., None]) -> None: action_dir: str = os.path.normpath(os.path.join(root, "ACTION", action_id)) state: AtomicListMap.Entry = g_CALLBACKS_PER_ID.add(f"ACTION/{action_id}", callback) if state != AtomicListMap.Entry.CREATED: # we are not first if state != AtomicListMap.Entry.INSERTED: # action ready, run callback callback(action_dir) return os.makedirs(action_dir) action_desc: Json = graph["actions"][action_id] num_deps = AtomicInt(len(action_desc.get("input", {}).items())) def run_command_and_callbacks() -> None: if num_deps.fetch_dec() <= 1: cmd: List[str] = action_desc["command"] env = action_desc.get("env") log("Running %r with env %r for action %r" % (cmd, env, action_id)) for out in action_desc["output"]: os.makedirs(os.path.join(action_dir, os.path.dirname(out)), exist_ok=True) exec_dir = action_dir if "cwd" in action_desc: exec_dir = os.path.join(action_dir, action_desc["cwd"]) subprocess.run(cmd, env=env, cwd=exec_dir, check=True) vals = g_CALLBACKS_PER_ID.fetch_clear(f"ACTION/{action_id}") if vals: for cb in vals: # mark ready ts.add(cb, action_dir) if num_deps.value == 0: run_command_and_callbacks() for location, desc in action_desc.get("input", {}).items(): def create_link(location: str) -> Callable[..., None]: def do_link(path: str) -> None: link(path, os.path.join(action_dir, location)) run_command_and_callbacks() return do_link ts.add(build, desc, config=config, root=root, graph=graph, ts=ts, callback=create_link(location)) def build_action(desc: Json, *, config: Json, root: str, graph: Json, ts: TaskSystem, callback: Callable[..., None]) -> None: def link_output(action_dir: str) -> None: callback(os.path.join(action_dir, desc["data"]["path"])) run_action(desc["data"]["id"], config=config, root=root, graph=graph, ts=ts, callback=link_output) def build(desc: Json, *, config: Json, root: str, graph: Json, ts: TaskSystem, callback: Callable[..., None]) -> None: if desc["type"] == "TREE": build_tree(desc, config=config, root=root, graph=graph, ts=ts, callback=callback) elif desc["type"] == "ACTION": build_action(desc, config=config, root=root, graph=graph, ts=ts, callback=callback) elif desc["type"] == "KNOWN": callback(build_known(desc, root=root)) elif desc["type"] == "LOCAL": callback(build_local(desc, root=root, config=config)) else: fail("Don't know how to build artifact %r" % (desc, )) def traverse(*, graph: Json, to_build: Json, out: str, root: str, config: Json) -> None: os.makedirs(out, exist_ok=True) os.makedirs(root, exist_ok=True) with TaskSystem() as ts: create_blobs(graph["blobs"], root=root, ts=ts) ts.finish() for location, artifact in to_build.items(): def create_link(location: str) -> Callable[..., None]: return lambda path: link(path, os.path.join(out, location)) ts.add(build, artifact, config=config, root=root, graph=graph, ts=ts, callback=create_link(location)) def main(): parser = ArgumentParser() parser.add_argument("-C", dest="repository_config", help="Repository-description file to use", metavar="FILE") parser.add_argument("-o", dest="output_directory", help="Directory to place output to") parser.add_argument("--local-build-root", dest="local_build_root", help="Root for storing intermediate outputs", metavar="PATH") parser.add_argument("--default-workspace", dest="default_workspace", help="Workspace root to use if none is specified", metavar="PATH") (options, args) = parser.parse_known_args() if len(args) != 2: fail("usage: %r " % (sys.argv[0], )) with open(args[0]) as f: graph: Json = json.load(f) with open(args[1]) as f: to_build: Json = json.load(f) out: str = os.path.abspath(cast(str, options.output_directory or "out-boot")) root: str = os.path.abspath( cast(str, options.local_build_root or ".just-boot")) with open(options.repository_config or "repo-conf.json") as f: config = json.load(f) if options.default_workspace: ws_root: str = os.path.abspath(options.default_workspace) repos: List[str] = config.get("repositories", {}).keys() for repo in repos: if not "workspace_root" in config["repositories"][repo]: config["repositories"][repo]["workspace_root"] = [ "file", ws_root ] traverse(graph=graph, to_build=to_build, out=out, root=root, config=config) if __name__ == "__main__": main() just-buildsystem-justbuild-b1fb5fa/doc/000077500000000000000000000000001516554100600203645ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/concepts/000077500000000000000000000000001516554100600222025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/concepts/alternative-mirrors.md000066400000000000000000000064561516554100600265500ustar00rootroot00000000000000Alternative Mirrors =================== Background ---------- The repository fetch tool `just-mr` reads a repository-configuration file that describes `git` repositories and repositories given by archives in a content-addressable way: by the `git` commit id, or the blob identifier of the archive, respectively. Hence it does not matter where these resources are fetched form; nevertheless, a location to fetch from is needed for `just-mr` to be able to set up those repositories. Typically the main upstream URL is given. While `just-mr` fetches each archive or commit only once and keeps a local copy, large organizations still tend to set up a local mirror. In this way, not every person working on the project has to fetch from upstream (thus reducing load there) and there is a central place archiving all the dependencies. Such an archive is needed anyway, for audit and compliance reasons, as well as to be able to continue the project independent of the dependencies' upstream. It is therefore desirable that `just-mr` support this workflow while still clearly pointing out the upstream location, e.g., for updating. Implementation -------------- We have introduced the following, backwards-compatible, extensions to `just-mr`. ### Mirrors field in `just-mr` repository config In the multi-repository configuration, in the definition of an `"archive"`, `"zip"` or `"git"` repository, an addition field `"mirrors"` was added. If given, this field has to be a list of URLs that provide additional places where the respective repository can be fetched from. `just-mr` will only consider a fetch failed if the repository cannot be fetched neither from the main location (described in `"fetch"` field for `"archive"` and `"zip"` repositories, and `"repository"` field for `"git"` repositories), nor from any of the specified mirrors. ### Additional mirrors in the just local specification Local mirrors of organizations are often not available to the general public and possibly not even available to everyone who has access to the respective project. In order to avoid polluting a multi-repository specification with the URLs of such restricted mirrors, the `.just-local` file was extended to support additional keys in its JSON object. - For the optional key `"local mirrors"`, if given, a JSON object is specified that maps primary URLs to a list of local (non-public) mirrors. Those mirrors are always tried first (in the given order) before any other URL is contacted. - For the optional key `"preferred hostnames"`, if given, a list of host names is specified. When fetching from a non-local mirror, URLs with host names in the given list are preferred (in the order given) over URLs with other host names. ### `just-import-git` to support mirror specifications As multi-repository specifications are often generated from a description of the local repositories by a chain of `just-import-git` invocations, this tool needs to be able to also insert a `"mirrors"` field for the repository imported. Therefore, this tool was extended by an option `--mirror` where multiple occurrences accumulate to specify additional mirrors for the URL fetched. These mirrors, if any, form the `"mirrors"` field of the repository imported; they are not used for the fetching in the import step itself, as the head commit of the primary URL is to be taken. just-buildsystem-justbuild-b1fb5fa/doc/concepts/anonymous-targets.md000066400000000000000000000344571516554100600262400ustar00rootroot00000000000000Anonymous targets ================= Motivation ---------- Using [Protocol buffers](https://github.com/protocolbuffers/protobuf) allows to specify, in a language-independent way, a wire format for structured data. This is done by using description files from which APIs for various languages can be generated. As protocol buffers can contain other protocol buffers, the description files themselves have a dependency structure. From a software-engineering point of view, the challenge is to ensure that the author of the description files does not have to be aware of the languages for which APIs will be generated later. In fact, the main benefit of the language-independent description is that clients in various languages can be implemented using the same wire protocol (and thus capable of communicating with the same server). For a build system that means that we have to expect that language bindings at places far away from the protocol definition, and potentially several times. Such a duplication can also occur implicitly if two buffers, for which language bindings are generated both use a common buffer for which bindings are never requested explicitly. Still, we want to avoid duplicate work for common parts and we have to avoid conflicts with duplicate symbols and staging conflicts for the libraries for the common part. Our approach is that a "proto" target only provides the description files together with their dependency structure. From those, a consuming target generates "anonymous targets" as additional dependencies; as those targets will have an appropriate notion of equality, no duplicate work is done and hence, as a side effect, staging or symbol conflicts are avoided as well. Preliminary remark: action identifiers -------------------------------------- Actions are defined as Merkle-tree hash of the contents. As all components (input tree, list of output strings, command vector, environment, and cache pragma) are given by expressions, that can quickly be computed. This identifier also defines the notion of equality for actions, and hence action artifacts. Recall that equality of artifacts is also (implicitly) used in our notion of disjoint map union (where the set of keys does not have to be disjoint, as long as the values for all duplicate keys are equal). When constructing the action graph for traversal, we can drop duplicates (i.e., actions with the same identifier, and hence the same description). For the serialization of the graph as part of the analyse command, we can afford the preparatory step to compute a map from action id to list of origins. Equality -------- ### Notions of equality In the context of builds, there are different concepts of equality to consider. We recall the definitions, as well as their use in our build tool. #### Locational equality ("Defined at the same place") Names (for targets and rules) are given by repository name, module name, and target name (inside the module); additionally, for target names, there's a bit specifying that we explicitly refer to a file. Names are equal if and only if the respective strings (and the file bit) are equal. For targets, we use locational equality, i.e., we consider targets equal precisely if their names are equal; targets defined at different places are considered different, even if they're defined in the same way. The reason we use notion of equality is that we have to refer to targets (and also check if we already have a pending task to analyse them) before we have fully explored them with all the targets referred to in their definition. #### Intensional equality ("Defined in the same way") In our expression language we handle definitions; in particular, we treat artifacts by their definition: a particular source file, the output of a particular action, etc. Hence we use intensional equality in our expression language; two objects are equal precisely if they are defined in the same way. This notion of equality is easy to determine without the need of reading a source file or running an action. We implement quick tests by keeping a Merkle-tree hash of all expression values. #### Extensional equality ("Defining the same object") For built artifacts, we use extensional equality, i.e., we consider two files equal, if they are bit-by-bit identical. Implementation-wise, we compare an appropriate cryptographic hash. Before running an action, we built its inputs. In particular (as inputs are considered extensionally) an action might cause a cache hit with an intensionally different one. #### Observable equality ("The defined objects behave in the same way") Finally, there is the notion of observable equality, i.e., the property that two binaries behaving the same way in all situations. As this notion is undecidable, it is never used directly by any build tool. However, it is often the motivation for a build in the first place: we want a binary that behaves in a particular way. ### Relation between these notions The notions of equality were introduced in order from most fine grained to most coarse. Targets defined at the same place are obviously defined in the same way. Intensionally equal artifacts create equal action graphs; here we can confidently say "equal" and not only isomorphic: due to our preliminary clean up, even the node names are equal. Making sure that equal actions produce bit-by-bit equal outputs is the realm of [reproducible builds](https://reproducible-builds.org/). The tool can support this by appropriate sandboxing, etc, but the rules still have to define actions that don't pick up non-input information like the current time, user id, readdir order, etc. Files that are bit-by-bit identical will behave in the same way. ### Example Consider the following target file. ```jsonc { "foo": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo Hello World > out.txt"] } , "bar": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo Hello World > out.txt"] } , "baz": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo -n Hello > out.txt && echo ' World' >> out.txt"] } , "foo upper": { "type": "generic" , "deps": ["foo"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "bar upper": { "type": "generic" , "deps": ["bar"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "baz upper": { "type": "generic" , "deps": ["baz"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "ALL": { "type": "install" , "files": {"foo.txt": "foo upper", "bar.txt": "bar upper", "baz.txt": "baz upper"} } } ``` Assume we build the target `"ALL"`. Then we will analyse 7 targets, all the locationally different ones (`"foo"`, `"bar"`, `"baz"`, `"foo upper"`, `"bar upper"`, `"baz upper"`). For the targets `"foo"` and `"bar"`, we immediately see that the definition is equal; their intensional equality also renders `"foo upper"` and `"bar upper"` intensionally equal. Our action graph will contain 4 actions: one with origins `["foo", "bar"]`, one with origins `["baz"]`, one with origins `["foo upper", "bar upper"]`, and one with origins `["baz upper"]`. The `"install"` target will, of course, not create any actions. Building sequentially (`-J 1`), we will get one cache hit. Even though the artifacts of `"foo"` and `"bar"` and of `"baz"` are defined differently, they are extensionally equal; both define a file with contents `"Hello World\n"`. Anonymous targets ----------------- Besides named targets we also have additional targets (and hence also configured targets) that are not associated with a location they are defined at. Due to the absence of definition location, their notion of equality will take care of the necessary deduplication (implicitly, by the way our dependency exploration works). We will call them "anonymous targets", even though, technically, they're not fully anonymous as the rules that are part of their structure will be given by name, i.e., defining rule location. ### Value type: target graph node In order to allow targets to adequately describe a dependency structure, we have a value type in our expression language, that of a (target) graph node. As with all value types, equality is intensional, i.e., nodes defined in the same way are equal even if defined at different places. This can be achieved by our usual approach for expressions of having cached Merkle-tree hashes and comparing them when an equality test is required. This efficient test for equality also allows using graph nodes as part of a map key, e.g., for our asynchronous map consumers. As a graph node can only be defined with all data given, the defined dependency structure is cycle-free by construction. However, the tree unfolding will usually be exponentially larger. For internal handling, this is not a problem: our shared-pointer implementation can efficiently represent a directed acyclic graph and since we cache hashes in expressions, we can compute the overall hash without folding the structure to a tree. When presenting nodes to the user, we only show the map of identifier to definition, to avoid that exponential unfolding. We have two kinds of nodes. #### Value nodes These represent a target that, in any configuration, returns a fixed value. Source files would typically be represented this way. The constructor function `"VALUE_NODE"` takes a single argument `"$1"` that has to be a result value. #### Abstract nodes These represent internal nodes in the dag. Their constructor `"ABSTRACT_NODE"` takes the following arguments (all evaluated). - `"node_type"`. An arbitrary string, not interpreted in any way, to indicate the role that the node has in the dependency structure. When we create an anonymous target from a node, this will serve as the key into the rule mapping to be applied. - `"string_fields"`. This has to be a map of strings. - `"target_fields"`. These have to be a map of lists of graph nodes. Moreover, we require that the keys for maps provided as `"string_fields"` and `"target_fields"` be disjoint. ### Graph nodes in `export` targets Graph nodes are completely free of names and hence are eligible for exporting. As with other values, in the cache the intensional definition of artifacts implicit in them will be replaced by the corresponding, extensionally equal, known value. However, some care has to be taken in the serialisation that is part of the caching, as we do not want to unfold the dag to a tree. Therefore, we take as JSON serialisation a simple dict with `"type"` set to `"NODE"`, and `"value"` set to the Merkle-tree hash. That serialisation respects intensional equality. To allow deserialisation, we add an additional map to the serialisation from node hash to its definition. ### Dependings on anonymous targets #### Parts of an anonymous target An anonymous target is given by a pair of a node and a map mapping the abstract node-type specifying strings to rule names. So, in the implementation these are just two expression pointers (with their defined notion of equality, i.e., equality of the respective Merkle-tree hashes). Such a pair of pointers also forms an additional variant of a name value, referring to such an anonymous target. It should be noted that such an anonymous target contains all the information needed to evaluate it in the same way as a regular (named) target defined by a user-defined rule. It is an analysis error analysing an anonymous target where there is no entry in the rules map for the string given as `"node_type"` for the corresponding node. #### Anonymous targets as additional dependencies We keep the property that a user can only request named targets. So anonymous targets have to be requested by other targets. We also keep the property that other targets are only requested at certain fixed steps in the evaluation of a target. To still achieve a meaningful use of anonymous targets our rule language handles anonymous targets in the following way. ##### Rules parameter `"anonymous"` In the rule definition a parameter `"anonymous"` (with empty map as default) is allowed. It is used to define an additional dependency on anonymous targets. The value has to be a map with keys the additional implicitly defined field names. It is hence a requirement that the set of keys be disjoint from all other field names (the values of `"config_fields"`, `"string_fields"`, and `"target_fields"`, as well as the keys of the `"implict"` parameter). Another consequence is that `"config_transitions"` map may now also have meaningful entries for the keys of the `"anonymous"` map. Each value in the map has to be itself a map, with entries `"target"`, `"provider"`, and `"rule_map"`. For `"target"`, a single string has to be specified, and the value has to be a member of the `"target_fields"` list. For provider, a single string has to be specified as well. The idea is that the nodes are collected from that provider of the targets in the specified target field. For `"rule_map"` a map has to be specified from strings to rule names; the latter are evaluated in the context of the rule definition. ###### Example For generating language bindings for protocol buffers, a rule might look as follows. ``` jsonc { "cc_proto_bindings": { "target_fields": ["proto_deps"] , "anonymous": { "protos": { "target": "proto_deps" , "provider": "proto" , "rule_map": {"proto_library": "cc_proto_library"} } } , "expression": {...} } } ``` ##### Evaluation mechanism The evaluation of a target defined by a user-defined rule is handled as follows. After the target fields are evaluated as usual, an additional step is carried out. For each anonymous-target field, i.e., for each key in the `"anonymous"` map, a list of anonymous targets is generated from the corresponding value: take all targets from the specified `"target"` field in all their specified configuration transitions (they have already been evaluated) and take the values provided for the specified `"provider"` key (using the empty list as default). That value has to be a list of nodes. All the node lists obtained that way are concatenated. The configuration transition for the respective field name is evaluated. Those targets are then evaluated for all the transitioned configurations requested. In the final evaluation of the defining expression, the anonymous-target fields are available in the same way as any other target field. Also, they contribute to the effective configuration in the same way as regular target fields. just-buildsystem-justbuild-b1fb5fa/doc/concepts/blob-splitting.md000066400000000000000000000260251516554100600254620ustar00rootroot00000000000000Blob Splitting Protocol ======================= Introduction ------------ Due to the generic nature of our build system, we do not impose any restrictions on the type of artifacts that are generated by actions or that are inputs to actions. Thus, artifacts might be large executables, libraries, or a compiled set of slides in PDF format, or even whole file system images. Such artifacts result in large blobs, which need to be fetched from a remote CAS if they are to be inspected or used locally. Depending on the network connection, this might imply a significant waiting time until the complete artifact is downloaded as well as results in a lot of network traffic. This situation is especially painful in case of short modification-inspection cycles. For each small modification of the sources, the complete artifact needs to be downloaded even though maybe only a small fraction of the compiled binary artifact has been changed. Thus, we introduced a blob splitting API as conservative extension to the original [remote execution API](https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto), which allows to split the binary data of a blob into chunks and then to transmit only the modified chunks instead of the whole blob data. This reduces traffic between the remote server and the local host and avoids unnecessary waiting times for users frequently working on large artifacts. The downside of this API is an increased storage consumption, since the binary data of the splitted artifacts will be stored twice. Remote execution API extension ------------------------------ We extended the existing remote execution API by introducing a new remote procedure call (rpc) to the `ContentAddressableStorage` service to split a blob into chunks along with respective request and response messages, and by adding a flag to the `CacheCapabilities` message to indicate whether blob splitting is supported by the server instance or not. The following code snippet depicts the required extensions to the remote execution protocol: service ContentAddressableStorage { ... rpc SplitBlob(SplitBlobRequest) returns (SplitBlobResponse) {} } message SplitBlobRequest { string instance_name = 1; Digest blob_digest = 2; } message SplitBlobResponse { repeated Digest chunk_digests = 1; } message CacheCapabilities { ... bool blob_split_support = ; } ### Contract between client and server The contract between the client and the server for this protocol extension is that if a client requests to split a blob at the remote execution server, the server answers with a list of chunk digests and gives the promise that first, all referenced blobs are available in its CAS to be downloaded and second, the concatenation of the returned blobs in the given order will result in the original blob the client has requested. The client does not give any promise to the server, it is free to not use the blob splitting rpc, but in order to make sense of the protocol extension (saving traffic), a meaningful behavior of the client would be to request the chunk digests of a blob from the server, to fetch only those chunks that are missing in its local CAS, and to store the fetched chunks as well as the reconstructed original blob in its local CAS. If the client requests to split a blob, but the server does not support blob splitting or if an error occurred during the request, the client falls back to fetch the entire blob. Blob split procedure -------------------- ### Server side When the server receives a request to split a blob via the `SplitBlob` rpc, it first checks whether the blob given by the `blob_digest` from the `SplitBlobRequest` message exists in its CAS. If not, the status code `google.rpc.Code.NOT_FOUND` is returned. Otherwise, it loads the blob data and splits it into chunks according to the implemented data chunking algorithm. As explained later, there are different implementation options for this algorithm, but they all must ensure one property: the concatenation of the chunks result in the original blob. After the blob is splitted, the server computes a digest for each chunk according to the configured digest function and puts each chunk that is not yet stored in its CAS. If an error occurs during storing the chunks in the CAS due to storage shortage, the status code `google.rpc.Code.RESOURCE_EXHAUSTED` is returned. Otherwise, the chunk digests are collected in the `chunk_digests` field of a `SplitBlobResponse` message in the same order as the chunks occur within the binary blob data and the message is sent back to the client with the status code `google.rpc.Code.OK`. > Note: Since the same data is basically stored twice for each blob, the > storage consumption for the remote server is roughly doubled. Some > savings occur when chunks are reused in more than one blob or even > several times in the same blob. ### Client side If a client wants to take advantage from blob splitting, it requests to split a blob into chunks at the remote side by calling the `SplitBlob` rpc from the `ContentAddressableStorage` service given a `SplitBlobRequest` message containing the `blob_digest` of the respective blob. If the status code returned by the `SplitBlob` rpc contains an error, the split operation failed and the client needs to fall back to fetch the entire blob. Otherwise, blob splitting was successful and the remote server returns an ordered list of `chunk_digests` and guarantees that the chunks are available in its CAS and that the concatenation of all chunks in the given order will result in the requested blob. The client checks which chunks are available locally, fetches only the locally missing chunks from the remote side via the `BatchReadBlobs` rpc or the `ByteStream` API, and assembles the requested blob by concatenating the binary data of its chunks. Finally, it stores the chunks as well as the resulting blob in its local CAS. > Note: Since the same data is basically stored twice for each blob, the > local storage consumption is roughly doubled. Some savings occur when > chunks are reused in more than one blob or even several times in the > same blob. Compatibility issues -------------------- We aim to get this protocol extension into the official remote execution API mentioned above. Once this is done and field numbers are assigned, servers that have upgraded to the new API version inform clients that have upgraded to the new API version whether they support blob splitting or not by setting the `blob_split_support` field in the `CacheCapabilities` message accordingly. A server only provides the aforementioned guarantee to clients once it has announced to support blob splitting using this field. An old client can communicate with a server implementing the new API without any modification. Nobody is forced to use the blob splitting functionality and unknown fields are just ignored at the client side. A client implementing the new API can communicate with an old server by evaluating the `blob_split_support` field, which will be set to its default value `false` at the client side. Until the protocol extension is officially accepted, the field number for the `blob_split_support` field is not known. In this case, early implementors can use a _trial request_ to determine whether the remote server supports blob splitting or not. This allows to implement a prototypical client and server employing blob splitting without requiring the protocol extension to be officially accepted. It also allows new clients to communicate with old servers, they still behave correctly and do not need to be adapted. The client implementation needs to be rebuilt once a field number has been assigned. Data chunking algorithm ----------------------- As stated earlier, a blob split request from a client is answered by the server with the promise that the concatenation of the returned chunks results in the requested blob. While this property is guaranteed by the server, it does not tell anything about the quality of the split result. It is desirable to generate chunks that are likely to be known by the client, because then less data needs to be transferred. A trivial implementation of the split operation would be returning a _singleton list_ containing the original blob as single chunk. While this implementation is correct and fast, it is not very useful, since it does not save any traffic. A better approach is _fixed-block chunking_ where the data is split into chunks of fixed and equal size. While this approach typically results in more than one chunk and improves the probability to save some traffic, it comes with the limitation of the data shifting problem. The insertion of a single character at the beginning of a data stream shifts the entire data while chunk boundaries are fixed. Thus, completely new chunks, unknown to the client are created even though the data patterns are mostly similar. A more intelligent way of splitting blobs would be to look for locations in the content of the data as chunk boundaries such that splitting of a slightly modified blob results in almost similar chunks insensitive to the data shifting problem of fixed-block chunking. Since the resulting chunk sizes are typically different, this approach is also called _variable-block chunking_ and is explained in the following section. ### Variable-block chunking In variable-block chunking, the borders of the chunks are content defined and shifted with the data pattern. This can be achieved by computing hash values of data blocks of a specific size at every byte position in the data stream and matching those hash values against a predefined pattern. In case of a match, that byte position is declared as chunk boundary. This approach avoids the data shifting problem of fixed-block chunking and ensures that unmodified data more than a block size away from the changes will have the same boundaries. A possible pattern to be matched might be all zeroes in the lower `n` bits. This would result in chunks with an average size of `2^n` bytes. ### Rolling hashes A common technique to efficiently compute hash values of consecutive bytes at every byte position in the data stream is to use a rolling hash function. A rolling hash function allows to cheaply compute the hash value of a chunk of bytes of a specific size at byte position `i` from the hash value of the data chunk of the same size at byte position `i-1`. Different plain rolling hash functions are available for implementation such as a [polynomial rolling hash](https://ieeexplore.ieee.org/document/5390135), the [Rabin fingerprint](http://www.cs.cmu.edu/~15-749/READINGS/optional/rabin1981.pdf), or a [cyclic polynomial rolling hash](https://dl.acm.org/doi/abs/10.1145/256163.256168) (also called Buzhash). However, the [fast rolling Gear hash algorithm](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia) (also called FastCDC) has been proven to be very compute efficient and faster than the other rolling hash algorithms specifically for content-based chunking of a stream of data while achieving similar deduplication ratios as the Rabin fingerprint. just-buildsystem-justbuild-b1fb5fa/doc/concepts/built-in-rules.md000066400000000000000000000243051516554100600254030ustar00rootroot00000000000000Built-in rules ============== Targets are defined in `TARGETS` files. Each target file is a single `JSON` object. If the target name is contained as a key in that object, the corresponding value defines the target; otherwise it is implicitly considered a source file. The target definition itself is a `JSON` object as well. The mandatory key `"type"` specifies the rule defining the target; the meaning of the remaining keys depends on the rule defining the target. There are a couple of rules built in, all named by a single string. The user can define additional rules (and, in fact, we expect the majority of targets to be defined by user-defined rules); referring to them in a qualified way (with module) will always refer to those even if new built-in rules are added later (as built-in rules will always be only named by a single string). The following rules are built in. Built-in rules can have a special syntax. `"export"` ---------- The `"export"` rule evaluates a given target in a specified configuration. More precisely, the field `"target"` has to name a single target (not a list of targets), the field `"flexible_config"` a list of strings, treated as variable names, and the field `"fixed_config"` has to be a map that is taken unevaluated. It is a requirement that the domain of the `"fixed_config"` and the `"flexible_config"` be disjoint. The optional fields `"doc"` and `"config_doc"` can be used to describe the target and the `"flexible_config"`, respectively. To evaluate an `"export"` target, first the configuration is restricted to the `"flexible_config"` and then the union with the `"fixed_config"` is built. The target specified in `"target"` is then evaluated. It is a requirement that this target be untainted. The result is the result of this evaluation; artifacts, runfiles, and provides map are forwarded unchanged. The main point of the `"export"` rule is, that the relevant part of the configuration can be determined without having to analyze the target itself. This makes such rules eligible for target-level caching (provided the content of the repository as well as all reachable ones can be determined cheaply). This eligibility is also the reason why it is good practice to only depend on `"export"` targets of other repositories. `"install"` ----------- The `"install"` rules allows to stage artifacts (and runfiles) of other targets in a different way. More precisely, a new stage (i.e., map of artifacts with keys treated as file names) is constructed in the following way. The runfiles from all targets in the `"deps"` field are taken; the `"deps"` field is an evaluated field and has to evaluate to a list of targets. It is an error, if those runfiles conflict. The `"files"` argument is a special form. It has to be a map, and the keys are taken as paths. The values are evaluated and have to evaluate to a single target. That target has to have a single artifact or no artifacts and a single run file. In this way, `"files"` defines a stage; this stage overlays the runfiles of the `"deps"` and conflicts are ignored. Finally, the `"dirs"` argument has to evaluate to a list of pairs (i.e., lists of length two) with the first argument a target name and the second argument a string, taken as directory name. For each entry, both, runfiles and artifacts of the specified target are staged to the specified directory. It is an error if a conflict with the stage constructed so far occurs. Both, runfiles and artifacts of the `"install"` target are the stage just described. An `"install"` target always has an empty provides map. Any provided information of the dependencies is discarded. `"generic"` ----------- The `"generic"` rules allows to define artifacts as the output of an action. This is mainly useful for ad-hoc constructions; for anything occurring more often, a proper user-defined rule is usually the better choice. The `"deps"` argument is evaluated and has to evaluate to a list of target names. The runfiles and artifacts of these targets form the inputs of the action. Conflicting definitions for individual paths are not an error and resolved by giving precedence to the artifacts over the runfiles; conflicts within artifacts or runfiles are resolved in a latest-wins fashion using the order of the targets in the evaluated `"deps"` argument. However, the input stage obtained by those resolution rules has to be free of semantic conflicts. The fields `"cmds"`, `"cwd"`, `"sh -c"`, `"out_dirs"`, `"outs"`, and `"env"` are evaluated fields where `"cmds"`, `"out_dirs"`, and `"outs"` have to evaluate to a list of strings, `"sh -c"` has to evaluate to a list of strings or `null`, `"env"` has to evaluate to a map of strings, and `"cwd"` has to evaluate to a single string naming a non-upwards relative path. During their evaluation, the functions `"outs"` and `"runfiles"` can be used to access the logical paths of the artifacts and runfiles, respectively, of a target specified in `"deps"`. Here, `"env"` specifies the environment in which the action is carried out. `"out_dirs"` and `"outs"` define the output directories and files, respectively, the action has to produce; these path are read relative to the action root. Since some artifacts are to be produced, at least one of `"out_dirs"` or `"outs"` must be a non-empty list of strings. It is an error if one or more paths are present in both the `"out_dirs"` and `"outs"`. Finally, the strings in `"cmds"` are extended by a newline character and joined, and command of the action is the result of evaluating the field `"sh -c"` (or `["sh", "-c"]` if `"sh -c"` evaluates to `null` or `[]`) extended by this string; the command is executed in the subdirectory of the execution root specified by `"cwd"`. The artifacts of this target are the outputs (as declared by `"out_dirs"` and `"outs"`) of this action. Runfiles and provider map are empty. `"file_gen"` ------------ The `"file_gen"` rule allows to specify a file with a given content. To be able to accurately report about file names of artifacts or runfiles of other targets, they can be specified in the field `"deps"` which has to evaluate to a list of targets. The names of the artifacts and runfiles of a target specified in `"deps"` can be accessed through the functions `"outs"` and `"runfiles"`, respectively, during the evaluation of the arguments `"name"` and `"data"` which have to evaluate to a single string. Artifacts and runfiles of a `"file_gen"` target are a singleton map with key the result of evaluating `"name"` and value a (non-executable) file with content the result of evaluating `"data"`. The provides map is empty. `"tree"` -------- The `"tree"` rule allows to specify a tree out of the artifact stage of given targets. More precisely, the deps field `"deps"` has to evaluate to a list of targets. For each target, runfiles and artifacts are overlayed in an artifacts-win fashion and the union of the resulting stages is taken; it is an error if conflicts arise in this way. The resulting stage is transformed into a tree. Both, artifacts and runfiles of the `"tree"` target are a singleton map with the key the result of evaluating `"name"` (which has to evaluate to a single string) and value that tree. `"tree_overlay"` and `"disjoint_tree_overlay"` ---------------------------------------------- The rules `"tree_overlay"` and `"disjoint_tree_overlay"` allow to define a tree as the overlay of the trees given by the artifact stages of a list of given targets. More precisely, the field `"deps"` has to evaluate to a list of targets. For each target, a tree is formed from its artifact stage. A new tree is defined as the tree overlay, or disjoint tree overlay, of those trees. Both, artifacts and runfiles of the target are a singleton map with key the result of evaluating the field `"name"` which has to evaluate to a single string and value that (disjoint) tree overlay. `"symlink"` ------------ The `"symlink"` rule allows to specify a non-upwards symbolic link with a given link target. To be able to accurately report about file names of artifacts or runfiles of other targets, they can be specified in the field `"deps"` which has to evaluate to a list of targets. The names of the artifacts and runfiles of a target specified in `"deps"` can be accessed through the functions `"outs"` and `"runfiles"`, respectively, during the evaluation of the arguments `"name"` and `"data"` which have to evaluate to a single string. Artifacts and runfiles of a `"symlink"` target are a singleton map with key the result of evaluating `"name"` and value a non-upwards symbolic link with target path the result of evaluating `"data"` (which must evaluate to a non-upwards path). The provides map is empty. `"configure"` ------------- The `"configure"` rule allows to configure a target with a given configuration. The field `"target"` is evaluated and the result of the evaluation must name a single target (not a list). The `"config"` field is evaluated and must result in a map, which is used as configuration overlay for the given target. This rule uses the given configuration overlay to modify the current environment for evaluating the given target, and thereby performs a configuration transition. It forwards all results (artifacts/runfiles/provides map) of the configured target to the upper context. The result of a target that uses this rule is the result of the target given in the `"target"` field (the configured target). As a full configuration transition is performed, the same care has to be taken when using this rule as when writing a configuration transition in a rule. Typically, this rule is used only at a top-level target of a project and configures only variables internally to the project. In any case, when using non-internal targets as dependencies (i.e., targets that a caller of the `"configure"` potentially might use as well), care should be taken that those are only used in the initial configuration. Such preservation of the configuration is necessary to avoid conflicts, if the targets depended upon are visible in the `"configure"` target itself, e.g., as link dependency (which almost always happens when depending on a library). Even if a non-internal target depended upon is not visible in the `"configure"` target itself, requesting it in a modified configuration causes additional overhead by increasing the target graph and potentially the action graph. just-buildsystem-justbuild-b1fb5fa/doc/concepts/cache-pragma.md000066400000000000000000000137351516554100600250450ustar00rootroot00000000000000Action caching pragma ===================== Introduction: exit code, build failures, and caching ---------------------------------------------------- The exit code of a process is used to signal success or failure of that process. By convention, 0 indicates success and any other value indicates some form of failure. Our tool expects all build actions to follow this convention. A non-zero exit code of a regular build action has two consequences. - As the action failed, the whole build is aborted and considered failed. - As such a failed action can never be part of a successful build, it is (effectively) not cached. This non-caching is achieved by rerequesting an action without cache look up, if a failed action from cache is reported. In particular, for building, we have the property that everything that does not lead to aborting the build can (and will) be cached. This property is justified as we expect build actions to behave in a functional way. Test and run actions -------------------- Tests have a lot of similarity to regular build actions: a process is run with given inputs, and the results are processed further (e.g., to create reports on test suites). However, they break the above described connection between caching and continuation of the build: we expect that some tests might be flaky (even though they shouldn't be, of course) and hence only want to cache successful tests. Nevertheless, we do want to continue testing after the first test failure. Another breakage of the functionality assumption of actions are "run" actions, i.e., local actions that are executed either because of their side effect on the host system, or because of their non-deterministic results (e.g., monitoring some resource). Those actions should never be cached, but if they fail, the build should be aborted. Tainting -------- Targets that, directly or indirectly, depend on non-functional actions are not regular targets. They are test targets, run targets, benchmark results, etc; in any case, they are tainted in some way. When adding high-level caching of targets, we will only support caching for untainted targets. To make everybody aware of their special nature, they are clearly marked as such: tainted targets not generated by a tainted rule (e.g., a test rule) have to explicitly state their taintedness in their attributes. This declaration also gives a natural way to mark targets that are technically pure, but still should be used only in test, e.g., a mock version of a larger library. Besides being for tests only, there might be other reasons why a target might not be fit for general use, e.g., configuration files with accounts for developer access, or files under restrictive licences. To avoid having to extend the framework for each new use case, we allow arbitrary strings as markers for the kind of taintedness of a target. Of course, a target can be tainted in more than one way. More precisely, rules can have `"tainted"` as an additional property. Moreover `"tainted"` is another reserved keyword for target arguments (like `"type"` and `"arguments_config"`). In both cases, the value has to be a list of strings, and the empty list is assumed, if not specified. A rule is tainted with the set of strings in its `"tainted"` property. A target is tainted with the union of the set of strings of its `"tainted"` argument and the set of strings its generating rule is tainted with. Every target has to be tainted with (at least) the union of what its dependencies are tainted with. For tainted targets, the `analyse`, `build`, and `install` commands report the set of strings the target is tainted with. ### `"may_fail"` and `"no_cache"` properties of `"ACTION"` The `"ACTION"` function in the defining expression of a rule have two additional (besides inputs, etc) parameters `"may_fail"` and `"no_cache"`. Those are not evaluated and have to be lists of strings (with empty assumed if the respective parameter is not present). Only strings the defining rule is tainted with may occur in that list. If the list is not empty, the corresponding may-fail or no-cache bit of the action is set. For actions with the `"may_fail"` bit set, the optional parameter `"fail_message"` with default value `"action failed"` is evaluated. That message will be reported if the action returns a non-zero exit value. Actions with the no-cache bit set are never cached. If an action with the may-fail bit set exits with non-zero exit value, the build is continued if the action nevertheless managed to produce all expected outputs. We continue to ignore actions with non-zero exit status from cache. ### Marking of failed artifacts To simplify finding failures in accumulated reports, our tool keeps track of artifacts generated by failed actions. More precisely, artifacts are considered failed if one of the following conditions applies. - Artifacts generated by failed actions are failed. - Tree artifacts containing a failed artifact are failed. - Artifacts generated by an action taking a failed artifact as input are failed. The identifiers used for built artifacts (including trees) remain unchanged; in particular, they will only describe the contents and not if they were obtained in a failed way. When reporting artifacts, e.g., in the log file, an additional marker is added to indicate that the artifact is a failed one. After every `build` or `install` command, if the requested artifacts contain failed one, a different exit code is returned. ### The `install-cas` subcommand A typical workflow for testing is to first run the full test suite and then only look at the failed tests in more details. As we don't take failed actions from cache, installing the output can't be done by rerunning the same target as `install` instead of `build`. Instead, the output has to be taken from CAS using the identifier shown in the build log. To simplify this workflow, there is the `install-cas` subcommand that installs a CAS entry, identified by the identifier as shown in the log to a given location or (if no location is specified) to `stdout`. just-buildsystem-justbuild-b1fb5fa/doc/concepts/computed-roots.md000066400000000000000000000223011516554100600255060ustar00rootroot00000000000000Computed roots ============== Use cases for computed build descriptions ----------------------------------------- ### Generated target files Sometimes projects (or parts thereof that can form a separate logical repository) have a simple structure. For example, there is a list of directories and for each one there is a library, named and staged in a systematic way. Repeating all those systematic target files seems unnecessary work. Instead, we could store the list of directories to consider and a small script containing the naming/staging/globbing logic; this approach would also be more maintainable. A similar approach could also be attractive for a directory tree with tests where, on top, all the individual tests should be collected to test suites. ### Staging according to embedded information For importing prebuilt libraries, it is sometimes desirable to stage them in a way honoring the embedded `soname`. The current approach is to provide that information out of band in the target file, so that it can be used during analysis. Still, the information is already present in the prebuilt binary, causing unnecessary maintenance overhead; instead, the target file could be a function of that library which can form its own content-fixed root (e.g., a `git tree` root), so that the computed value is easily cacheable. ### Simplified rule definition and alternative syntax Rules can share computation through expressions. However, the interface, deliberately has to be explicit, including the documentation strings that are used by `just describe`. While this allows easy and efficient implementation of `just describe`, there is some redundancy involved, as often fields are only there to be used by a common expression, but this have to be documented in a redundant way (causing additional maintenance burden). Moreover, using JSON encoding of abstract syntax trees is an unambiguously readable and easy to automatically process format, but people argue that it is hard to write by hand. However, it is unlikely to get agreement on which syntax is best to use. Now, if rule and expression files can be generated, this argument is not necessary. Moreover, rules are typically versioned and infrequently changed, so the step of generating the official syntax from the convenient one would typically be in cache. Root types depending on computation ----------------------------------- There are two additional types of roots that are defined through computation. They allow a clean principle to add the needed (and a lot more) flexibility for the described use cases, while ensuring that all computations of roots are properly cacheable at high level. In this way, we do not compromise efficient builds, as the price of the additional flexibility; in the typical case, is just a single cache lookup. Of course, it is up to the user to ensure that this case really is the typical one, in the same way as it is their responsibility to describe the targets in a way to have proper incrementality. ### Root type `"computed"` The `just` multi-repository configuration allows a type of root, called `"computed"`. A `"computed"` root is given by - the (global) name of a repository - the name of a target (in `["module", "target"]` format), and - a configuration (as JSON object, taken literally). It is a requirement that the specified target is an `"export"` target and the specified repository content-fixed; `"computed"` roots are considered content-fixed. However, the dependency structure of computed roots must be cycle free. In other words, there must exist an ordering of computed roots (the implicit topological order, not a declared one) such that for each computed root, the referenced repository as well as all repositories reachable from that one via the `"bindings"` map only contain computed roots earlier in that order. ### Root type `"tree structure"` In the described use case of generated target files, the tree of target files only depends on the structure of the workspace root. To avoid unnecessary actions, an additional root type is defined, that of a `"tree structure"`. Such a root is given by precisely one root. It evaluates to that root but with all files replaced by empty files. Obviously, this computation can be done without spawning actions and is cachable. The serve functionality also allows to answer queries for the tree structure of a given tree known to serve. ### Strict evaluation of roots as artifact tree The building of required computed roots happens in topological order; the build of the defining target of a root is, in principle (subject to a user-defined restriction of parallelism) started as soon as all roots in the repositories reachable via bindings are available. The root is then considered the artifact tree of the defining target. In particular, the evaluation is strict: all roots of reachable repositories have to be successfully computed before the evaluation is started, even if it later turns out that one of these roots is never accessed in the computation of the defining target. The reason for this strictness requirement is to ensure that the cache key for target-level caching can be computed ahead of time (and we expect the entry to be in target-level cache most of the time anyway). ### Intensional equality of computed roots During a build, each computed root is evaluated only once, even if required in several places. Two computed roots are considered equal, if they are defined in the same way, i.e., repository name, target, and configuration agree. The repository or layer using the computed root is not part of the root definition. Similarly, two tree-structure roots are equal if the defining roots are equal. ### Evaluation through serve endpoint preferred When determining the value of a computed root, as for every export target, the provided serve endpoint (if any) is consulted first. Only if it is not aware of the root, a local evaluation is carried out. This strategy is also applied for tree-structure roots. ### `just-mr` support for computed roots To allow simply setting up a `just` configuration using computed roots, `just-mr` allows a repository type `"computed"` with the same parameters as a computed root, as well as a repository type `"tree structure"` with another root as parameter. These repositories can be used as roots, like any other `just-mr` repository type. When generating the `just` multi-repository configuration, the definition of a `"computed"` repository is just forwarded as computed root. ### Computed roots and `just serve` Due to the presence of `just serve`, roots can be absent. This affects computed roots in two ways, - roots, in particular the target roots, of the repository referred to can be absent, and - a computed root can be absent itself. The latter has to be supported, as dependencies that should be delegated to `just serve` might contain computed roots themselves. In this case, we consider it acceptable to have one round of talking back and forth with the serve instance per computed root involved, however we do not want to fetch the artifacts of those intermediate roots. After all, whole point of the serve service was to use dependencies without having them locally. #### Syntax for absent computed roots As for other roots, we let the user specify which roots are to be absent. Tools like `just-import-git` will extend their marking of absent dependencies (e.g., by the option `--absent` of `just-import-git`) to computed roots as well. In a `just-mr` repository config, `"pragma": {"absent": true}` can be used for computed roots as well. Also `just-mr` will also honor the passed absent specification (via `--absent` or implicitly via the rc file) for computed roots the same way as for other roots. In a `just` repository config, computed roots are given by the tuple `["computed", , , , ]`. Optionally, an additional entry can be added; that entry has to be an object. A computed root is absent if that additional argument is present and contains an entry for the value `"absent"` that is `true`. E.g., `["computed", "base", "", "", {}]` is a concrete computed root and `["computed", "base", "", "", {}, {"absent": true}]` is the same computed root considered absent. #### Evaluation of computed roots in connection with absent roots If a computed root is absent then, in native mode, regardless of whether the base repository is absent or not, - serve will be asked for the result, and - from the result the tree identifier of the root will be computed in memory and the root set to that value, as absent. If a concrete computed root refers to a base repository with absent target root, - the client will ask serve about the flexible variables of the specified target, and - with this information will compute locally the cache key and inspect the local target-level cache. If not there, the root will be built, installed to a local temporary directory and imported into the git cas. In the remaining case of a concrete computed root with concrete target root of the referred base repository, the cache key can be computed locally and a local check for a cache hit can be performed; in this way, unnecessary IO-operations are avoided. If no cache hit is found, the target will be built, installed to a temporary directory and imported into the git cas. just-buildsystem-justbuild-b1fb5fa/doc/concepts/configuration.md000066400000000000000000000105361516554100600254000ustar00rootroot00000000000000Configuration ============= Targets describe abstract concepts like "library". Depending on requirements, a library might manifest itself in different ways. For example, - it can be built for various target architectures, - it can have the requirement to produce position-independent code, - it can be a special build for debugging, profiling, etc. So, a target (like a library described by header files, source files, dependencies, etc) has some additional input. As those inputs are typically of a global nature (e.g., a profiling build usually wants all involved libraries to be built for profiling), this additional input, called "configuration" follows the same approach as the `UNIX` environment: it is a global collection of key-value pairs and every target picks, what it needs. Top-level configuration ----------------------- The configuration is a `JSON` object. The configuration for the target requested can be specified on the command line using the `-c` option; its argument is a file name and that file is supposed to contain the `JSON` object. Propagation ----------- Rules and target definitions have to declare which parts of the configuration they want to have access to. The (essentially) full configuration, however, is passed on to the dependencies; in this way, a target not using a part of the configuration can still depend on it, if one of its dependencies does. ### Rules configuration and configuration transitions As part of the definition of a rule, it specifies a set `"config_vars"` of variables. During the evaluation of the rule, the configuration restricted to those variables (variables unset in the original configuration are set to `null`) is used as environment. Additionally, the rule can request that certain targets be evaluated in a modified configuration by specifying `"config_transitions"` accordingly. Typically, this is done when a tool is required during the build; then this tool has to be built for the architecture on which the build is carried out and not the target architecture. Those tools often are `"implicit"` dependencies, i.e., dependencies that every target defined by that rule has, without the need to specify it in the target definition. ### Target configuration Additionally (and independently of the configuration-dependency of the rule), the target definition itself can depend on the configuration. This can happen, if a debug version of a library has additional dependencies (e.g., for structured debug logs). If such a configuration-dependency is needed, the reserved key word `"arguments_config"` is used to specify a set of variables (if unset, the empty set is assumed; this should be the usual case). The environment in which all arguments of the target definition are evaluated is the configuration restricted to those variables (again, with values unset in the original configuration set to `null`). For example, a library where the debug version has an additional dependency could look as follows. ``` jsonc { "libfoo": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["DEBUG"] , "name": ["foo"] , "hdrs": ["foo.hpp"] , "srcs": ["foo.cpp"] , "local defines": { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["DEBUG"] } , "deps": { "type": "++" , "$1": [ ["libbar", "libbaz"] , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["libdebuglog"] } ] } } } ``` Effective configuration ----------------------- A target is influenced by the configuration through - the configuration dependency of target definition, as specified in `"arguments_config"`, - the configuration dependency of the underlying rule, as specified in the rule's `"config_vars"` field, and - the configuration dependency of target dependencies, not taking into account values explicitly set by a configuration transition. Restricting the configuration to this collection of variables yields the effective configuration for that target-configuration pair. The `--dump-targets` option of the `analyse` subcommand allows to inspect the effective configurations of all involved targets. Due to configuration transitions, a target can be analyzed in more than one configuration, e.g., if a library is used both, for a tool needed during the build, as well as for the final binary cross-compiled for a different target architecture. just-buildsystem-justbuild-b1fb5fa/doc/concepts/doc-strings.md000066400000000000000000000153111516554100600247610ustar00rootroot00000000000000Documentation of build rules, expressions, etc ============================================== Build rules can obtain a non-trivial complexity. This is especially true if several rules have to exist for slightly different use cases, or if the rule supports many different fields. Therefore, documentation of the rules (and also expressions for the benefit of rule authors) is desirable. Experience shows that documentation that is not versioned together with the code it refers to quickly gets out of date, or lost. Therefore, we add documentation directly into the respective definitions. Multi-line strings in JSON -------------------------- In JSON, the newline character is encoded specially and not taken literally; also, there is not implicit joining of string literals. So, in order to also have documentation readable in the JSON representation itself, instead of single strings, we take arrays of strings, with the understanding that they describe the strings obtained by joining the entries with newline characters. Documentation is optional ------------------------- While documentation is highly recommended, it still remains optional. Therefore, when in the following we state that a key is for a list or a map, it is always implied that it may be absent; in this case, the empty array or the empty map is taken as default, respectively. Rules ----- Each rule is described as a JSON object with a fixed set of keys. So having fixed keys for documentation does not cause conflicts. More precisely, the keys `"doc"`, `"field_doc"`, `"config_doc"`, `"artifacts_doc"`, `"runfiles_doc"`, and `"provides_doc"` are reserved for documentation. Here, `"doc"` has to be a list of strings describing the rule in general. `"field_doc"` has to be a map from (some of) the field names to an array of strings, containing additional information on that particular field. `"config_doc"` has to be a map from (some of) the config variables to an array of strings describing the respective variable. `"artifacts_doc"` is an array of strings describing the artifacts produced by the rule. `"runfiles_doc"` is an array of strings describing the runfiles produced by this rule. Finally, `"provides_doc"` is a map describing (some of) the providers by that rule; as opposed to fields or config variables there is no authoritative list of providers given elsewhere in the rule, so it is up to the rule author to give an accurate documentation on the provided data. ### Example ``` jsonc { "library": { "doc": [ "A C library" , "" , "Define a library that can be used to be statically linked to a" , "binary. To do so, the target can simply be specified in the deps" , "field of a binary; it can also be a dependency of another library" , "and the information is then propagated to the corresponding binary." ] , "string_fields": ["name"] , "target_fields": ["srcs", "hdrs", "private-hdrs", "deps"] , "field_doc": { "name": ["The base name of the library (i.e., the name without the leading lib)."] , "srcs": ["The source files (i.e., *.c files) of the library."] , "hdrs": [ "The public header files of this library. Targets depending on" , "this library will have access to those header files" ] , "private-hdrs": [ "Additional internal header files that are used when compiling" , "the source files. Targets depending on this library have no access" , "to those header files." ] , "deps": [ "Any other libraries that this library uses. The dependency is" , "also propagated (via the link-deps provider) to any consumers of" , "this target. So only direct dependencies should be declared." ] } , "config_vars": ["CC"] , "config_doc": { "CC": [ "single string. defaulting to \"cc\", specifying the compiler" , "to be used. The compiler is also used to launch the preprocessor." ] } , "artifacts_doc": ["The actual library (libname.a) staged in the specified directory"] , "runfiles_doc": ["The public headers of this library"] , "provides_doc": { "compile-deps": [ "Map of artifacts specifying any additional files that, besides the runfiles," , "have to be present in compile actions of targets depending on this library" ] , "link-deps": [ "Map of artifacts specifying any additional files that, besides the artifacts," , "have to be present in a link actions of targets depending on this library" ] , "link-args": [ "List of strings that have to be added to the command line for linking actions" , "in targets depending on this library" ] } , "expression": { ... } } } ``` Expressions ----------- Expressions are also described by a JSON object with a fixed set of keys. Here we use the keys `"doc"` and `"vars_doc"` for documentation, where `"doc"` is an array of strings describing the expression as a whole and `"vars_doc"` is a map from (some of) the `"vars"` to an array of strings describing this variable. Export targets -------------- As export targets play the role of interfaces between repositories, it is important that they be documented as well. Again, export targets are described as a JSON object with fixed set of keys and we use the keys `"doc"` and `"config_doc"` for documentation. Here `"doc"` is an array of strings describing the targeted in general and `"config_doc"` is a map from (some of) the variables of the `"flexible_config"` to an array of strings describing this parameter. Configure targets ----------------- As configure targets often serve as internal interface to external export targets (e.g., in order to set a needed configuration), we support documentation here as well. As configure targets, being built-in, have a fixed set of fields, a `"doc"` field can be used for this purpose without conflicts. Again, the `"doc"` field is an array of strings describing the target in general. Presentation of the documentation --------------------------------- As all documentation are just values (that need not be evaluated) in JSON objects, it is easy to write tool rendering documentation pages for rules, etc, and we expect those tools to be written independently. Nevertheless, for the benefit of developers using rules from a git-tree roots that might not be checked out, there is a subcommand `describe` which takes a target specification like the `analyze` command, looks up the corresponding rule and describes it fully, i.e., prints in human-readable form - the documentation for the rule - all the fields available for that rule together with - their type (`"string_fields"`, `"target_fields"`, etc), and - their documentation, - all the configuration variables of the rule with their documentation (if given), and - the documented providers. just-buildsystem-justbuild-b1fb5fa/doc/concepts/execution-properties.md000066400000000000000000000124471516554100600267310ustar00rootroot00000000000000Action-controlled execution properties ====================================== Motivation ---------- ### Varying execution platforms It is a common situation that software is developed for one platform, but it is desirable to build on a different one. For example, the other platform could be faster (common theme when developing for embedded devices), cheaper, or simply available in larger quantities. The standard solution for these kind of situations is cross compiling: the binary is completely built on one platform, while being intended to run on a different one. This can be achieved by constructing the compiler invocations accordingly and is already built into our rules (at least for `C` and `C++`). The situation changes, however, once testing (especially end-to-end testing) comes into play. Here, we actually have to run the built binary---and do so on the target architecture. Nevertheless, we still want to offload as much as possible of the work to the other platform and perform only the actual test execution on the target platform. This requires a single build executing actions on two (or more) platforms. ### Varying execution times #### Calls to foreign build systems Often, third-party dependencies that natively build with a different build system and don't change too often (yet often enough to not have them part of the build image) are simply put in a single action, so that they get built only once, and then stay in cache for everyone. This is precisely, what our `rules-cc` rules like `["CC/foreign/make", "library"]` and `["CC/foreign/cmake", "library"]` do. For those compound actions, we of course expect them to run longer than normal actions that only consist of a single compiler or linker invocation. Giving an absolute amount of time needed for such an action is not reasonable, as that very much depends on the underlying hardware. However, it is reasonable to give a number "typical" actions this compound action corresponds to. #### Long-running end-to-end tests A similar situation where a significantly longer action is needed in a build otherwise consisting of short actions are end-to-end tests. Test using the final binary might have a complex set up, potentially involving several instances running to test communication, and require a lengthy sequence of interactions to get into the situation that is to be tested, or to verify the absence of degrading of the service under high load or extended usage. Interfaces related to action-controlled execution properties ------------------------------------------------------------ ### Properties of the `"ACTION"` function The `"ACTION"` function available in the rule definition also has the following attributes. All of those attributes are optional. #### `"execution properties"` This value has to evaluate to a map of strings or `null`; if not given or evaluating to `null`, the empty map is taken as default. This map is taken as a union with any remote-execution properties specified at the invocation of the build (if keys are defined both, for the entire build and in `"execution properties"` of a specific action, the latter takes precedence). Local execution continues to any execution properties specified. However, with the dispatch functionality of `just` described later, such execution properties can also influence a build that is local by default. #### `"timeout scaling"` If given, the value has to evaluate to a number greater or equal than `1.0`, or `null`. If not given, or evaluating to `null`, the value `1.0` is taken as default. The action timeout specified for this build (the default value, or whatever is specified on the command line) is multiplied by the given factor and taken as timeout for this action. This applies for both, local and remote builds. ### Execution properties of the the built-in `"generic"` rule As the built-in `"generic"` rule basically is there to allow the definition of an action in an ad-hoc fashion, it also provides the same attributes. More precisely, the fields `"timeout scaling"` and `"execution properties"` are taken as additional arguments to the underlying action, with the same semantics as the respective fields of the `"ACTION"` constructor. ### `just` dispatching based on remote-execution properties In simple setups, like using `just execute`, the remote execution is not capable of dispatching to different workers based on remote-execution properties. To nevertheless have the benefits of using different execution environments, `just` allows an optional configuration file to be passed on the command line via a new option `--endpoint-configuration`. This configuration file contains a list of pairs of remote-execution properties and remote-execution endpoints. The first matching entry (i.e., the first entry where the remote-execution property map coincides with the given map when restricted to its domain) determines the remote-execution endpoint to be used; if no entry matches, the default remote-execution endpoint is used. In any case, the remote-execution properties are forwarded to the chosen remote-execution endpoint without modification. When connecting a non-standard remote-execution endpoint, `just` will ensure that the applicable CAS of that endpoint will have all the needed artifacts for that action. It will also transfer all result artifacts back to the CAS of the default remote-execution endpoint. just-buildsystem-justbuild-b1fb5fa/doc/concepts/expressions.md000066400000000000000000000473421516554100600251200ustar00rootroot00000000000000Expression language =================== At various places, in particular in order to define a rule, we need a restricted form of functional computation. This is achieved by our expression language. Syntax ------ All expressions are given by JSON values. One can think of expressions as abstract syntax trees serialized to JSON; nevertheless, the precise semantics is given by the evaluation mechanism described later. Semantic Values --------------- Expressions evaluate to semantic values. Semantic values are JSON values extended by additional atomic values for build-internal values like artifacts, names, etc. ### Truth Every value can be treated as a boolean condition. We follow a convention similar to `LISP` considering everything true that is not empty. More precisely, the values - `null`, - `false`, - `0`, - `""`, - the empty map, and - the empty list are considered logically false. All other values are logically true. Evaluation ---------- The evaluation follows a strict, functional, call-by-value evaluation mechanism; the precise evaluation is as follows. - Atomic values (`null`, booleans, strings, numbers) evaluate to themselves. - For lists, each entry is evaluated in the order they occur in the list; the result of the evaluation is the list of the results. - For JSON objects (which can be understood as maps, or dicts), the key `"type"` has to be present and has to be a literal string. That string determines the syntactical construct (sloppily also referred to as "function") the object represents, and the remaining evaluation depends on the syntactical construct. The syntactical construct has to be either one of the built-in ones or a special function available in the given context (e.g., `"ACTION"` within the expression defining a rule). All evaluation happens in an "environment" which is a map from strings to semantic values. ### Built-in syntactical constructs #### Special forms ##### Variables: `"var"` There has to be a key `"name"` that (i.e., the expression in the object at that key) has to be a literal string, taken as variable name. If the variable name is in the domain of the environment and the value of the environment at the variable name is non-`null`, then the result of the evaluation is the value of the variable in the environment. Otherwise, the key `"default"` is taken (if present, otherwise the value `null` is taken as default for `"default"`) and evaluated. The value obtained this way is the result of the evaluation. ##### Quoting: `"'"` The value is the value of the key `"$1"` uninterpreted, if present, and `null` otherwise. ##### Quasi-Quoting: ``"`"`` The value is the value of the key `"$1"` uninterpreted but replacing all outermost maps having a key `"type"` with the value either `","` or `",@"` in the following way. - If the value for the key `"type"` is `","`, the value for the key `"$1"` (or `null` if there is no key `"$1"`) is evaluated and the map is replaced by the value of that evaluation. - If the value for the key `"type"` is `",@"` it is an error if that map is not an entry of a literal list. The value for the key `"$1"` (or `[]` if there is no key `"$1"`) is evaluated; the result of that evaluation has to be a list. The entries of that list (i.e., the list obtained by evaluating the value for `"$1"`) are inserted (not the list itself) into the surrounding list replacing that map. For example, ``{"type": "`", "$1": [1, 2, {"type": ",@", "$1": [3, 4]}]}`` evaluates to `[1, 2, 3, 4]` while ``{"type": "`", "$1": [1, 2, {"type": ",", "$1": [3, 4]}]}`` evaluates to `[1, 2, [3, 4]]`. ##### Sequential binding: `"let*"` The key `"bindings"` (default `[]`) has to be (syntactically) a list of pairs (i.e., lists of length two) with the first component a literal string. For each pair in `"bindings"` the second component is evaluated, in the order the pairs occur. After each evaluation, a new environment is taken for the subsequent evaluations; the new environment is like the old one but amended at the position given by the first component of the pair to now map to the value just obtained. Finally, the `"body"` is evaluated in the final environment (after evaluating all binding entries) and the result of evaluating the `"body"` is the value for the whole `"let*"` expression. ##### Environment Map: `"env"` Creates a map from selected environment variables. The key `"vars"` (default `[]`) has to be a list of literal strings referring to the variable names that should be included in the produced map. This field is not evaluated. This expression is only for convenience and does not give new expression power. It is equivalent but lot shorter to multiple `singleton_map` expressions combined with `map_union`. ##### Conditionals ###### Binary conditional: `"if"` First the key `"cond"` is evaluated. If it evaluates to a value that is logically true, then the key `"then"` (if present, otherwise `[]` is taken as default) is evaluated and its value is the result of the evaluation. Otherwise, the key `"else"` (if present, otherwise `[]` is taken as default) is evaluated and the obtained value is the result of the evaluation. ###### Sequential conditional: `"cond"` The key `"cond"` has to be a list of pairs. In the order of the list, the first components of the pairs are evaluated, until one evaluates to a value that is logically true. For that pair, the second component is evaluated and the result of this evaluation is the result of the `"cond"` expression. If all first components evaluate to a value that is logically false, the result of the expression is the result of evaluating the key `"default"` (defaulting to `[]`). ###### String case distinction: `"case"` If the key `"case"` is present, it has to be a map (an "object", in JSON's terminology). In this case, the key `"expr"` is evaluated; it has to evaluate to a string. If the value is a key in the `"case"` map, the expression at this key is evaluated and the result of that evaluation is the value for the `"case"` expression. Otherwise (i.e., if `"case"` is absent or `"expr"` evaluates to a string that is not a key in `"case"`), the key `"default"` (with default `[]`) is evaluated and this gives the result of the `"case"` expression. ###### Sequential case distinction on arbitrary values: `"case*"` If the key `"case"` is present, it has to be a list of pairs. In this case, the key `"expr"` is evaluated. It is an error if that evaluates to a name-containing value. The result of that evaluation is sequentially compared to the evaluation of the first components of the `"case"` list until an equal value is found. In this case, the evaluation of the second component of the pair is the value of the `"case*"` expression. If the `"case"` key is absent, or no equality is found, the result of the `"case*"` expression is the result of evaluating the `"default"` key (with default `[]`). ##### Conjunction and disjunction: `"and"` and `"or"` For conjunction, if the key `"$1"` (with default `[]`) is syntactically a list, its entries are sequentially evaluated until a logically false value is found; in that case, the result is `false`, otherwise true. If the key `"$1"` has a different shape, it is evaluated and has to evaluate to a list. The result is the conjunction of the logical values of the entries. In particular, `{"type": "and"}` evaluates to `true`. For disjunction, the evaluation mechanism is the same, but the truth values and connective are taken dually. So, `"and"` and `"or"` are logical conjunction and disjunction, respectively, using short-cut evaluation if syntactically admissible (i.e., if the argument is syntactically a list). ##### Mapping ###### Mapping over lists: `"foreach"` First the key `"range"` is evaluated and has to evaluate to a list. For each entry of this list, the expression `"body"` is evaluated in an environment that is obtained from the original one by setting the value for the variable specified at the key `"var"` (which has to be a literal string, default `"_"`) to that value. The result is the list of those evaluation results. ###### Mapping over maps: `"foreach_map"` Here, `"range"` has to evaluate to a map. For each entry (in lexicographic order (according to native byte order) by keys), the expression `"body"` is evaluated in an environment obtained from the original one by setting the variables specified at `"var_key"` and `"var_val"` (literal strings, default values `"_"` and `"$_"`, respectively). The result of the evaluation is the list of those values. ##### Zipping ###### `"zip_with"` The keys `"range_1"` and `"range_2"` are evaluated and have to evaluate to lists. For each pair of entries, one from each list, in order, the expression `"body"` is evaluated in an environment obtained from the original one by correspondingly setting the variables specified at `"var_1"` and `"var_2"` (default values `"$1"` and `"$2"`, respectively). The result of the evaluation is the list of those values. If the input lists are of different lengths, any entry with no correspondence in the other list is ignored. ###### `"zip_map"` The keys `"range_key"` and `"range_val"` are evaluated and have to evaluate to lists. The result is a map, from an entry in `"range_key"` to the entry in `"range_val"` with the same index. If the input lists are of different lengths, any entry with no correspondence in the other list is ignored. It is equivalent but more convenient and more performant to a `"zip_with"` expression generating a list of maps combined with a `map_union`. ##### Folding: `"foldl"` The key `"range"` is evaluated and has to evaluate to a list. Starting from the result of evaluating `"start"` (default `[]`) a new value is obtained for each entry of the range list by evaluating `"body"` in an environment obtained from the original by binding the variable specified by `"var"` (literal string, default `"_"`) to the list entry and the variable specified by `"accum_var"` (literal string, default value `"$1"`) to the old value. The result is the last value obtained. #### Regular functions First `"$1"` is evaluated; for binary functions `"$2"` is evaluated next. For functions that accept keyword arguments, those are evaluated as well. Finally the function is applied to this (or those) argument(s) to obtain the final result. ##### Unary functions - `"not"` Return the logical negation of the argument, i.e., if the argument is logically false, return `true`, and `false` otherwise. - `"nub_right"` The argument has to be a list. It is an error if that list contains (directly or indirectly) a name. The result is the input list, except that for all duplicate values, all but the rightmost occurrence is removed. - `"nub_left"` The argument has to be a list. It is an error if that list contains (directly or indirectly) a name. The result is the input list, except that for all duplicate values, all but the leftmost occurrence is removed. - `"basename"` The argument has to be a string. This string is interpreted as a path, and the file name thereof is returned. - `"keys"` The argument has to be a map. The result is the list of keys of this map, in lexicographical order (according to native byte order). - `"values"` The argument has to be a map. The result are the values of that map, ordered by the corresponding keys (lexicographically according to native byte order). - `"range"` The argument is interpreted as a non-negative integer as follows. Non-negative numbers are rounded to the nearest integer; strings have to be the decimal representation of an integer; everything else is considered zero. The result is a list of the given length, consisting of the decimal representations of the first non-negative integers. For example, `{"type": "range", "$1": "3"}` evaluates to `["0", "1", "2"]`. - `"enumerate"` The argument has to be a list. The result is a map containing one entry for each element of the list. The key is the decimal representation of the position in the list (starting from `0`), padded with leading zeros to length at least 10. The value is the element. The padding is chosen in such a way that iterating over the resulting map (which happens in lexicographic order of the keys) has the same iteration order as the list for all lists indexable by 32-bit integers. - `"set"` The argument has to be a list of strings. The result is a map with the members of the list as keys, and all values being `true`. - `"reverse"` The argument has to be a list. The result is a new list with the entries in reverse order. - `"length"` The argument has to be a list. The result is the length of the list. - `"++"` The argument has to be a list of lists. The result is the concatenation of those lists. - `"+"` The argument has to be a list of numbers. The result is their sum (where the sum of the empty list is, of course, the neutral element 0). - `"*"` The argument has to be a list of numbers. The result is their product (where the product of the empty list is, of course, the neutral element 1). - `"map_union"` The argument has to be a list of maps. The result is a map containing as keys the union of the keys of the maps in that list. For each key, the value is the value of that key in the last map in the list that contains that key. - `"join_cmd"` The argument has to be a list of strings. A single string is returned that quotes the original vector in a way understandable by a POSIX shell. As the command for an action is directly given by an argument vector, `"join_cmd"` is typically only used for generated scripts. - `"json_encode"` The result is a single string that is the canonical JSON encoding of the argument (with minimal white space); all atomic values that are not part of JSON (i.e., the added atomic values to represent build-internal values) are serialized as `null`. ##### Unary functions with keyword arguments - `"change_ending"` The argument has to be a string, interpreted as path. The ending is replaced by the value of the keyword argument `"ending"` (a string, default `""`). For example, `{"type": "change_ending", "$1": "foo/bar.c", "ending": ".o"}` evaluates to `"foo/bar.o"`. - `"join"` The argument has to be a list of strings. The return value is the concatenation of those strings, separated by the the specified `"separator"` (strings, default `""`). - `"escape_chars"` Prefix every in the argument every character occurring in `"chars"` (a string, default `""`) by `"escape_prefix"` (a strings, default `"\"`). - `"to_subdir"` The argument has to be a map (not necessarily of artifacts). The keys as well as the `"subdir"` (string, default `"."`) argument are interpreted as paths and keys are replaced by the path concatenation of those two paths. If the optional argument `"flat"` (default `false`) evaluates to a true value, the keys are instead replaced by the path concatenation of the `"subdir"` argument and the base name of the old key. It is an error if conflicts occur in this way; in case of such a user error, the argument `"msg"` is also evaluated and the result of that evaluation reported in the error message. Note that conflicts can also occur in non-flat staging if two keys are different as strings, but name the same path (like `"foo.txt"` and `"./foo.txt"`), and are assigned different values. It also is an error if the values for keys in conflicting positions are name-containing. - `"from_subdir"` The argument has to be a map (not necessarily of artifacts). The keys of this map, as well as the value of keyword argument `"subdir"` (string, default `"."`) are interpreted as paths; only those key-value pairs of the argument map are kept where the key refers to an entry in the specified `"subdir"`, and for those the path relative to the subdir is taken as new key. Those paths relative to the subdir are taken in canonical form; it is an error if non-trivial conflicts arise that way, i.e., if two keys that are kept normalize to the same relative path while the respective values are different. ##### Binary functions - `"=="` The result is `true` is the arguments are equal, `false` otherwise. It is an error if one of the arguments are name-containing values. - `"concat_target_name"` This function is only present to simplify transitions from some other build systems and normally not used outside code generated by transition tools. The second argument has to be a string or a list of strings (in the latter case, it is treated as strings by concatenating the entries). If the first argument is a string, the result is the concatenation of those two strings. If the first argument is a list of strings, the result is that list with the second argument concatenated to the last entry of that list (if any). ##### Other functions - `"empty_map"` This function takes no arguments and always returns an empty map. - `"singleton_map"` This function takes two keyword arguments, `"key"` and `"value"` and returns a map with one entry, mapping the given key to the given value. - `"lookup"` This function takes two keyword arguments, `"key"` and `"map"`. The `"key"` argument has to evaluate to a string and the `"map"` argument has to evaluate to a map. If that map contains the given key and the corresponding value is non-`null`, the value is returned. Otherwise the `"default"` argument (with default `null`) is evaluated and returned. - `"[]"` This function takes two keyword arguments, `"index"` and `"list"`. The `"list"` argument has to evaluate to a list. The `"index"` argument has to evaluate to either a number (which is then rounded to the nearest integer) or a string (which is interpreted as an integer). If the index obtained in this way is valid for the obtained list, the entry at that index is returned; negative indices count from the end of the list. Otherwise the `"default"` argument (with default `null`) is evaluated and returned. #### Constructs related to reporting of user errors Normally, if an error occurs during the evaluation the error is reported together with a stack trace. This, however, might not be the most informative way to present a problem to the user, especially if the underlying problem is a proper user error, e.g., in rule usage (leaving out mandatory arguments, violating semantical prerequisites, etc). To allow proper error reporting, the following functions are available. All of them have an optional argument `"msg"` that is evaluated (only) in case of error and the result of that evaluation included in the error message presented to the user. - `"fail"` Evaluation of this function unconditionally fails. - `"context"` This function is only there to provide additional information in case of error. Otherwise it is the identify function (a unary function, i.e., the result of the evaluation is the result of evaluating the argument `"$1"`). - `"assert_non_empty"` Evaluate the argument (given by the parameter `"$1"`). If it evaluates to a non-empty string, map, or list, return the result of the evaluation. Otherwise fail. - `"disjoint_map_union"` Like `"map_union"` but it is an error, if two (or more) maps contain the same key, but map it to different values. It is also an error if the argument is a name-containing value. - `"assert"` Evaluate the argument (given by the parameter `"$1"`); then evaluate the expression `"predicate"` with the variable given at the key `"var"` (which has to be a string literal if given, default value is `"_"`) bound to that value. If the predicate evaluates to a true value, return the result of evaluating the argument, otherwise fail; in evaluating the failure message `"msg"`, also keep the variable specified by `"var"` bound to the result of evaluating the argument. just-buildsystem-justbuild-b1fb5fa/doc/concepts/garbage.md000066400000000000000000000256611516554100600241260ustar00rootroot00000000000000Garbage Collection ================== For every build, for all non-failed actions an entry is created in the action cache and the corresponding artifacts are stored in the CAS. So, over time, a lot of files accumulate in the local build root. Hence we have a way to reclaim disk space while keeping the benefits of having a cache. This operation is referred to as garbage collection and usually uses the heuristics to keeping what is most recently used. Our approach follows this paradigm as well. Invariants assumed by our build system -------------------------------------- Our tool assumes several invariants on the local build root, that we need to maintain during garbage collection. Those are the following. - If an artifact is referenced in any cache entry (action cache, target-level cache), then the corresponding artifact is in CAS. - If a tree is in CAS, then so are its immediate parts (and hence also all transitive parts). Generations of cache and CAS ---------------------------- In order to allow garbage collection while keeping the desired invariants, we keep several (currently two) generations of cache and CAS. Each generation in itself has to fulfill the invariants. The effective cache or CAS is the union of the caches or CASes of all generations, respectively. Obviously, then the effective cache and CAS fulfill the invariants as well. The actual `gc` command rotates the generations: the oldest generation is be removed and the remaining generations are moved one number up (i.e., currently the young generation will simply become the old generation), implicitly creating a new, empty, youngest generation. As an empty generation fulfills the required invariants, this operation preservers the requirement that each generation individually fulfill the invariants. All additions are made to the youngest generation; in order to keep the invariant, relevant entries only present in an older generation are also added to the youngest generation first. Moreover, whenever an entry is referenced in any way (cache hit, request for an entry to be in CAS) and is only present in an older generation, it is also added to the younger generation, again adding referenced parts first. As a consequence, the youngest generation contains everything directly or indirectly referenced since the last garbage collection; in particular, everything referenced since the last garbage collection will remain in the effective cache or CAS upon the next garbage collection. These generations are stored as separate directories inside the local build root. As the local build root is, starting from an empty directory, entirely managed by \`just\` and compatible tools, generations are on the same file system. Therefore the adding of old entries to the youngest generation can be implemented in an efficient way by using hard links. The moving up of generations can happen atomically by renaming the respective directory. Also, the oldest generation can be removed logically by renaming a directory to a name that is not searched for when looking for existing generations. The actual recursive removal from the file system can then happen in a separate step without any requirements on order. Parallel operations in the presence of garbage collection --------------------------------------------------------- The addition to cache and CAS can continue to happen in parallel; that certain values are taken from an older generation instead of freshly computed does not make a difference for the youngest generation (which is the only generation modified). But build processes assume they don't violate the invariant if they first add files to CAS and later a tree or cache entry referencing them. This, however, only holds true if no generation rotation happens in between. To avoid those kind of races, we make processes coordinate over a single lock for each build root. - Any build process keeps a shared lock for the entirety of the build. - The garbage collection process takes an exclusive lock for the period it does the directory renames. We consider it acceptable that, in theory, local build processes could starve local garbage collection. Moreover, it should be noted that the actual removal of no-longer-needed files from the file system happens without any lock being held. Hence the disturbance of builds caused by garbage collection is small. Compactification as part of garbage collection ---------------------------------------------- When building locally, all intermediate artifacts end up in the local CAS. Depending on the nature of the artifacts built, this can include large ones (like disk images) that only differ in small parts from the ones created in previous builds for a code base with only small changes. In this way, the disk can fill up quite quickly. Reclaiming this disk space by the described generation rotation would limit how far the cache reaches back. Therefore, large blobs are split in a content-defined way; when storing the chunks the large blobs are composed of, duplicate blobs have to be stored only once. ### Large-objects CAS Large objects are stored in a separated from of CAS, called large-objects CAS. It follows the same generation regime as the regular CAS; more precisely, next to the `casf`, `casx`, and `cast` two additional entries are generated, `cas-large-f` and `cas-large-t`, where the latter is only available in native mode (i.e., when trees are hashed differently than blobs). The entries in the large-objects CAS are keyed by hash of the large object and the value of an entry is the concatenation of the hashes of the chunks the large object is composed of. An entry in a large-object CAS promises - that the chunks the large object is composed of are in the main CAS, more precisely `casf` in the same generation, - the concatenation of the specified chunks indeed gives the requested object, - if the object is a tree, the parts are also in the same generation, in main or larger-object CAS, and - the object is strictly larger than the maximal size a chunk obtained by splitting can have. Here, the last promise avoids that chunks of a large object later can be replaced by a large-object entry themselves. ### Using objects from the large-objects CAS Whenever an object is not found in the main CAS, the large-objects CAS is inspected. If found there, then, in this order, - if the entry is not already in the youngest generation, the chunks are promoted to the youngest generation, - the object itself is spliced on disk in a temporary file, - if the object is a tree, the parts are promoted to the youngest generation (only necessary if the large-object entry was not found in the youngest generation anyway), - the large object is added to the respective main CAS, and finally - the large-objects entry is added to the youngest generation (if not already there). The promoting of the chunks ensures that they are already present in the youngest generation at almost no storage cost, as promoting is implemented using hard links. Therefore, the overhead of a later splitting of that blob is minimal. ### Blob splitting uses large-objects CAS as cache When `just execute` is asked to split an object, first the large-objects CAS is inspected. If the entry is present in the youngest generation, the answer is directly served from there; if found in an older generation, it is served from there after appropriate promotion (chunks; parts of the tree, if the object to split was a tree; large-objects entry) to the youngest generation. When `just execute` actually performs a split and the object that was to be splitted was larger than the maximal size of a chunk, after having added the chunks to the main CAS, it will also write a corresponding entry to the large-objects CAS. In this way, subsequent requests for the same object can be served from there without having to inspect the object again. Similarly, if a blob is split in order to be transferred to an endpoint that supports blob-splicing, a corresponding entry is added to the large-objects CAS (provided the original blob was larger than the maximal size of a chunk, but we will transfer by blob splice only for those objects anyway). ### Compactification of a CAS generation During garbage collection, while already holding the exclusive garbage-collection lock, the following compactification steps are performed on the youngest generation before doing the generation rotation. - For every entry in the large-objects CAS the corresponding entries in the main CAS are removed (for an entry in `cas-large-f` the entries in both, `casf` and `casx` are removed, as files and executable files are both hashed the same way, as blobs). - For every entry in the main CAS that is larger than the compactification threshold, the object is split, the chunks are added to the main CAS, the list of chunks the object is composed of is added to the large-objects CAS, and finally the object is removed from the main CAS. It should be noted that these steps do not modify the objects that can be obtained from that CAS generation. In particular, all invariants are kept. As compactification threshold we chose a fixed value larger than the maximal size a chunk obtained from splitting can have. More precisely, we take the maximal value where we still can transfer a blob via `grpc` without having to use the streaming interface, i.e, we chose 2MB. In this way, we already have correct blobs split for transfer to an end point that supports blob splicing. The compactification step will also be carried out if the `--no-rotate` option is given to `gc`. The compactification step is skipped if the `--all` option is given to `gc`, since that option triggers removal of all cache generations. `--no-rotate` and `--all` are incompatible options. Garbage Collection for Repository Roots --------------------------------------- The multi-repository tool `just-mr` often has to create roots: the tree for an archive, an explicit `"git tree"` root, etc. All those roots are stored in a `git` repository in the local build root. They are fixed by a tagged commit to be persistent there. In this way, the roots are available long-term and the association between archive hash and resulting root can be persisted. The actual archive can eventually be garbage collected as the root can efficiently be provided by that cached association. While this setup is good at preserving roots in a quite compact form, there currently is no mechanism to get rid of roots that are no longer needed. Especially switching between projects that have a large number of third-party dependencies, or on projects changing their set of dependencies frequently, this `git` repository in the local build root can grow large. Therefore, the repository roots follow a similar generation regime. The subcommand `gc-repo` of `just-mr` rotates generations and removes the oldest one. Whenever an entry is not found in the youngest generation of the repository-root storage, older generations are inspected first before calling out to the network; entries found in older generations are promoted to the youngest. just-buildsystem-justbuild-b1fb5fa/doc/concepts/multi-repo.md000066400000000000000000000170511516554100600246250ustar00rootroot00000000000000Multi-repository build ====================== Repository configuration ------------------------ ### Open repository names A repository can have external dependencies. This is realized by having unbound ("open") repository names being used as references. The actual definition of those external repositories is not part of the repository; we think of them as inputs, i.e., we think of this repository as a function of the referenced external targets. ### Binding in a separate repository configuration The actual binding of the free repository names is specified in a separate repository-configuration file, which is specified on the command line (via the `-C` option); this command-line argument is optional and the default is that the repository worked on has no external dependencies. Typically (but not necessarily), this repository-configuration file is located outside the referenced repositories and versioned separately or generated from such a file via `just-mr`. It serves as meta-data for a group of repositories belonging together. This file contains one JSON object. For the key `"repositories"` the value is an object; its keys are the global names of the specified repositories. For each repository, there is an object describing it. The key `"workspace_root"` describes where to find the repository and should be present for all (direct or indirect) external dependencies of the repository worked upon. Additional roots file names (for target, rule, and expression) can be specified. For keys not given, the same rules for default values apply as for the corresponding command-line arguments. Additionally, for each repository, the key "bindings" specifies the map of the open repository names to the global names that provide these dependencies. Repositories may depend on each other (or even themselves), but the resulting global target graph has to be cycle free. Whenever a location has to be specified, the value has to be a list, with the first entry being specifying the naming scheme; the semantics of the remaining entries depends on the scheme (see "Root Naming Schemes" below). Additionally, the optional key `"main"` specifies the main repository (if omitted, the repository with the lexicographically-first name is used). The target to be built (as specified on the command line) is taken from this repository. Also, the command-line arguments `-w`, `--target_root`, etc, apply to this repository. If no option `-w` is given and `"workspace_root"` is not specified in the repository-configuration file either, the root is determined from the working directory as usual. The value of `main` can be overwritten on the command line (with the `--main` option) In this way, a consistent configuration of interdependent repositories can be versioned and referred to regardless of the repository worked on. #### Root naming scheme ##### `"file"` The `"file"` scheme tells that the repository (or respective root) can be found in a directory in the local file system; the only argument is the absolute path to that directory. ##### `"git tree"` The `"git tree"` scheme tells that the root is defined to be a tree given by a git tree identifier. It takes two arguments - the tree identifier, as hex-encoded string, and - the absolute path to some repository containing that tree #### Example Consider, for example, the following repository-configuration file. In the following, we assume it is located at `/etc/just/repos.json`. ``` jsonc { "main": "env" , "repositories": { "foobar": { "workspace_root": ["file", "/opt/foobar/repo"] , "rule_root": ["file", "/etc/just/rules"] , "bindings": {"base": "barimpl"} } , "barimpl": { "workspace_root": ["file", "/opt/barimpl"] , "target_file_name": "TARGETS.bar" } , "env": {"bindings": {"foo": "foobar", "bar": "barimpl"}} } } ``` It specifies 3 repositories, with global names `foobar`, `barimpl`, and `env`. Within `foobar`, the repository name `base` refers to `barimpl`, the repository that can be found at `/opt/barimpl`. The repository `env` is the main repository and there is no workspace root defined for it, so it only provides bindings for external repositories `foo` and `bar`, but the actual repository is taken from the working directory (unless `-w` is specified). In this way, it provides an environment for developing applications based on `foo` and `bar`. For example, the invocation `just build -C /etc/just/repos.conf baz` tells our tool to build the target `baz` from the module the working directory is located in. `foo` will refer to the repository found at `/opt/foobar/repo` (using rules from `/etc/just/rules`, taking `base` refer to the repository at `/opt/barimpl`) and `bar` will refer to the repository at `/opts/barimpl`. Naming of targets ----------------- ### Reference in target files In addition to the normal target references (string for a target in the name module, module-target pair for a target in same repository, `["./", relpath, target]` relative addressing, `["FILE", null, name]` explicit file reference in the same module), references of the form `["@", repo, module, target]` can be specified, where `repo` is string referring to an open name. That open repository name is resolved to the global name by the `"bindings"` parameter of the repository the target reference is made in. Within the repository the resolved name refers to, the target `[module, target]` is taken. ### Expression language: names as abstract values Targets are a global concept as they distinguish targets from different repositories. Their names, however, depend on the repository they occur in (as the local names might differ in various repositories). Moreover, some targets cannot be named in certain repositories as not every repository has a local name in every other repository. To handle this naming problem, we note the following. During the evaluation of a target names occur at two places: as the result of evaluating the parameters (for target fields) and in the evaluation of the defining expression when requesting properties of a target dependent upon (via `DEP_ARTIFACTS` and related functions). In the later case, however, the only legitimate way to obtain a target name is by the `FIELD` function. To enforce this behavior, and to avoid problems with serializing target names, our expression language considers target names as opaque values. More precisely, - in a target description, the target fields are evaluated and the result of the evaluation is parsed, in the context of the module the `TARGET` file belongs to, as a target name, and - during evaluation of the defining expression of a the target's rule, when accessing `FIELD` the values of target fields will be reported as abstract name values and when querying values of dependencies (via `DEP_ARTIFACTS` etc) the correct abstract target name has to be provided. While the defining expression has access to target names (via target fields), it is not useful to provide them in provided data; a consuming data cannot use names unless it has those fields as dependency anyway. Our tool will not enforce this policy; however, only targets not having names in their provided data are eligible to be used in `export` rules. File layout in actions ---------------------- As `just` does full staging for actions, no special considerations are needed when combining targets of different repositories. Each target brings its staging of artifacts as usual. In particular, no repository names (neither local nor global ones) will ever be visible in any action. So for the consuming target it makes no difference if its dependency comes from the same or a different repository. just-buildsystem-justbuild-b1fb5fa/doc/concepts/overview.md000066400000000000000000000251571516554100600244040ustar00rootroot00000000000000Tool Overview ============= Structuring ----------- ### Structuring the Build: Targets, Rules, and Actions The primary units this build system deals with are targets: the user requests the system to build (or install) a target, targets depend on other targets, etc. Targets typically reflect the units a software developer thinks in: libraries, binaries, etc. The definition of a target only describes the information directly belonging to the target, e.g., its source, private and public header files, and its direct dependencies. Any other information needed to build a target (like the public header files of an indirect dependency) are inferred by the build tool. In this way, the build description can be kept maintainable A built target consists of files logically belonging together (like the actual library file and its public headers) as well as information on how to use the target (linking arguments, transitive header files, etc). For a consumer of a target, the definition of this collection of files as well as the additionally provided information is what defines the target as a dependency, irrespectively of where the target is coming from (i.e., targets coinciding here are indistinguishable for other targets). Of course, to actually build a single target from its dependencies, many invocations of the compiler or other tools are necessary (so called "actions"); the build tool translates these high-level description into the individual actions necessary and only re-executes those where inputs have changed. This translation of high-level concepts into individual actions is not hard coded into the tool. It is provided by the user as "rules" and forms additional input to the build. To avoid duplicate work, rules are typically maintained centrally for a project or an organization. ### Structuring the Code: Modules and Repositories The code base is usually split into many directories, each containing source files belonging together. To allow the definition of targets where their code is, the targets are structured in a similar way. For each directory, there can be a targets file. Directories for which such a targets file exists are called "modules". Each file belongs to the module that is closest when searching upwards in the directory tree. The targets file of a module defines the targets formed from the source files belonging to this module. Larger projects are often split into "repositories". For this build tool, a repository is a logical unit. Often those coincide with the repositories in the sense of version control. This, however, does not have to be the case. Also, from one directory in the file system many repositories can be formed that might differ in the rules used, targets defined, or binding of their dependencies. Staging ------- A peculiarity of this build system is the complete separation between physical and logical paths. Targets have their own view of the world, i.e., they can place their artifacts at any logical path they like, and this is how they look to other targets. It is up to the consuming targets what they do with artifacts of the targets they depend on; in particular, they are not obliged to leave them at the logical location their dependency put them. When such a collection of artifacts at logical locations (often referred to as the "stage") is realized on the file system (when installing a target, or as inputs to actions), the paths are interpreted as paths relative to the respective root (installation or action directory). This separation is what allows flexible combination of targets from various sources without leaking repository names or different file arrangement if a target is in the "main" repository. Repository data --------------- A repository uses a (logical) directory for several purposes: to obtain source files, to read definitions of targets, to read rules, and to read expressions that can be used by rules. While all those directories can be (and often are) the same, this does not have to be the case. For each of those purposes, a different logical directory (also called "root") can be used. In this way, one can, e.g., add target definitions to a source tree originally written for a different build tool without modifying the original source tree. Those roots are usually defined in a repository configuration. For the "main" repository, i.e., the repository from which the target to be built is requested, the roots can also be overwritten at the command line. Roots can be defined as paths in the file system, but also as `git` tree identifiers (together with the location of some repository containing that tree). The latter definition is preferable for rules and dependencies, as it allows high-level caching of targets. It also motivates the need of adding target definitions without changing the root itself. The same flexibility as for the roots is also present for the names of the files defining targets, rules, and expressions. While the default names `TARGETS`, `RULES`, and `EXPRESSIONS` are often used, other file names can be specified for those as well, either in the repository configuration or (for the main repository) on the command line. The final piece of data needed to describe a repository is the binding of the open repository names that are used to refer to other repositories. More details can be found in the documentation on multi-repository builds. Targets ------- ### Target naming In description files, targets, rules, and expressions are referred to by name. As the context always fixes if a name for a target, rule, or expression is expected, they use the same naming scheme. - A single string refers to the target with this name in the same module. - A pair `[module, name]` refers to the target `name` in the module `module` of the same repository. There are no module names with a distinguished meaning. The naming scheme is unambiguous, as all other names given by lists have length at least 3. - A list `["./", relative-module-path, name]` refers to a target with the given name in the module that has the specified path relative to the current module (in the current repository). - A list `["@", repository, module, name]` refers to the target with the specified name in the specified module of the specified repository. Additionally, there are special targets that can also be referred to in target files. - An explicit reference of a source-file target in the same module, specified as `["FILE", null, name]`. The explicit `null` at the second position (where normally the module would be) is necessary to ensure the name has length more than 2 to distinguish it from a reference to the module `"FILE"`. - An explicit reference of a non-upwards symlink target in the same module, specified as `["SYMLINK", null, name]`. The explicit `null` at the second position is required for the same reason as in the explicit file reference. It is the user's responsibility to ensure the symlink pointed to is non-upwards. - A reference to an collection, given by a shell pattern, of explicit source files in the top-level directory of the same module, specified as `["GLOB", null, pattern]`. The explicit `null` at second position is required for the same reason as in the explicit file reference. - A reference to a tree target in the same module, specified as `["TREE", null, name]`. The explicit `null` at second position is required for the same reason as in the explicit file reference. ### Data of an analyzed target Analyzing a target results in 3 pieces of data. - The "artifacts" are a staged collection of artifacts. Typically, these are what is normally considered the main reason to build a target, e.g., the actual library file in case of a library. - The "runfiles" are another staged collection of artifacts. Typically, these are files that directly belong to the target and are somehow needed to use the target. For example, in case of a library that would be the public header files of the library itself. - A "provides" map with additional information the target wants to provide to its consumers. The data contained in that map can also contain additional artifacts. Typically, this the remaining information needed to use the target in a build. In case of a library, that typically would include any other libraries this library transitively depends upon (a stage), the correct linking order (a list of strings), and the public headers of the transitive dependencies (another stage). A target is completely determined by these 3 pieces of data. A consumer of the target will have no other information available. Hence it is crucial, that everything (apart from artifacts and runfiles) needed to build against that target is contained in the provides map. When the installation of a target is requested on the command line, artifacts and runfiles are installed; in case of staging conflicts, artifacts take precedence. ### Source targets #### Files If a target is not found in the targets file, it is implicitly treated as a source file. Both, explicit and implicit source files look the same. The artifacts stage has a single entry: the path is the relative path of the file to the module root and the value the file artifact located at the specified location. The runfiles are the same as the artifacts and the provides map is empty. #### (Non-upwards) Symlinks To ensure self-containedness and location-independence, only *non-upwards* symlinks are expected and accepted. The symlinks must not however be necessarily resolvable, i.e., dangling symlinks are accepted. An explicit (non-upwards) symlink target is similar to an explicit file target, except that at the specified location there has to be a non-upwards symlink rather than a file and the corresponding symlink artifact is taken instead of a file artifact. #### Collection of files given by a shell pattern A collection of files given by a shell pattern has, both as artifacts and runfiles, the (necessarily disjoint) union of the artifact maps of the (zero or more) source targets that match the pattern. Only *files* in the *top-level* directory of the given modules are considered for matches. The provides map is empty. #### Trees A tree describes a directory. Internally, however, it is a single opaque artifact. Consuming targets cannot look into the internal structure of that tree. Only when realized in the file system (when installation is requested or as part of the input to an action), the directory structure is visible again. An explicit tree target is similar to an explicit file target, except that at the specified location there has to be a directory rather than a file and the tree artifact corresponding to that directory is taken instead of a file artifact. just-buildsystem-justbuild-b1fb5fa/doc/concepts/profiling.md000066400000000000000000000253271516554100600245260ustar00rootroot00000000000000# Profiling builds ## Use cases for logging build internals There are several use cases where inspection of the internals of a build can be provide additional insights. ### Time spent during the build For large projects, build times can grow quite large. While increased parallelism, e.g., via remote execution, helps to reduce build times, it cannot reduce build time below that of the critical path. So it is an important step in maintaining a code base to keep the critical path short, both in terms of number of actions, but also, more importantly in terms of typical runtime of those actions. Such a task requires profiling data of real-world builds. ### Test history When running test actions, beside the time spent it is also important to understand how a test status evolved over the history of the main branch of a code base, in terms of pass/fail/flaky, possibly also in comparison with test results on merge requests. In this way, information on the reliability and usefulness of the test can be deduced. It can also serve as basis for identifying when a test broke on the head branch, helping to identify the offending commit. ### Build-invocation sharing A different use case is to share a particular interesting build invocation with a different developer working on the same project. This can be a particularly interesting failure of a test (e.g., one that is flaky with extremely low error probability) or a hard-to-understand build error. Having all the information about that particular build invocation can facilitate cooperation, even with remote sites. ## Underlying design considerations In the described profiling use cases, the importance is evolution over time. For build steps, the relevant data is the time taken and the frequency that build step has to run. For steps that are only run very infrequently, usually the time taken is not that important. But for both those pieces of information, the evolution (typically along the main development branch of the code base) is relevant. - If a build step suddenly has to be run a lot more often, and hence gets more relevant for the overall duration of the build, this can be a hint that a change in the dependency structure added some hot parts of the code base to the transitive dependencies. Comparison of the dependency structure before and after that change can help to restructure the code in a way more favourable for efficient builds. - The the time for a single build step quickly increases over the history of the main branch, that is a candidate for a refactoring in the near future. Thus, by appropriately monitoring where the time for individual build steps increases, appropriate action can be taken before the time becomes a problem. Depending on the use case, a different form of accumulation of the observed data is needed. As it is infeasible to cover all the possible derived statistics, we instead focus only on providing the raw data of a single invocation. We leave it to the user create a suitable tool collecting the invocation data, possibly from different machines, and computing the relevant statistics as well as presenting it in a friendly way. In order to keep things simple, for the time being, we only support one form of outputting the needed data: writing it to a file. The collection and transfer to a central database is a task that can be solved by a separate tool (or by using `just add-to-cas` to collect in the remote-execution CAS). Nevertheless, we chose paths in such a way, that log files can be written to a network file system, which is one possible way of central collection. In any case, all output files are in a machine-readable format. Collection of build data, if desired, should be done routinely. For the build-sharing use case it is not known ahead of time, which invocation will be the interesting one. Also, statistics over invocations are much more informative, if the data is complete (or at least sampled in an unbiased way). Therefore, the build-data collection should be configured in a configuration file. The only tool we have that routinely reads a configuration file is `just-mr`. As this is also routinely used as a launcher for `just`, its configuration file is a good place to configure build-insight logging. Following the current design, we let `just-mr` do all the necessary set up and let `just` strictly follow instructions. ## Relevant interfaces ### The `--profile` option of `just` The build tool `just` has an option `--profile` with one file name as parameter. This option is accepted by all `just` subcommands that depend on analysis-related options (including `"describe"`). After completing the attempt for the requested task, it writes to the specified file name a file containing information about that attempt. This also happens if no build was attempted despite being requested, e.g., due to failure in analysis. The file contains a single JSON object, with the following key (and more keys possibly added in the future). - The key `"exit code"` contains the exit value of the `just` process; this allows easy filtering on the build and test results. - The key `"target"` contains the target in full-qualified form. The reason we include the target is that `just` allows to also deduce it from the invocation context (like working directory). - The key `"configuration"` contains the configuration in which this target is built. The reason this is included is that it is a value derived not only from the command line but also from the context of a file (given via `-c`) possibly local on the machine `just` was run. - The key `"remote"` describes the remote-execution endpoint, including the used properties and dispatch list. This allow distinguishing builds in different environments (possibly using different hardware); this can be relevant both for performance as well as failure statistics for tests. Again, the reason for including it in the profile is that local files are read (for the dispatch list). - The key `"actions"` contains a JSON object with the keys being precisely the key identifiers of the actions attempted to run. The action identifiers are the same as in the `--dump-graph` or `--dump-plain-graph` option; it is recommended to also run one of those options to have additional information on what those actions are. For each action attempted to run, the following is recorded (with possible additions in the future). - The exit code, as number. - The blob/tree identifier of all declared outputs, as well those of stdout/stderr (if given). This is the information needed for sharing a build invocation. However, it is also useful for running additional analysis tools over the actions causing lengthy builds; those additional analysis tools can answer questions like "Which input files precisely are the ones that change frequently?" and "Are those header files even used for that particular compilation step?". In case of tests, the collected outputs can be used to compare successful and failing runs of a flaky test. - Wether the action was cached. - For non-cached actions, the duration of the action, as floating-point seconds. If possible, that information is taken from the execution response (following the key `result` to the `ExecutedActionMetaData`); in this case, also additional meta data is recorded. If obtaining the meta data that way is not possible, the wall-clock time between starting and completing the action can be taken; in this case, the fact that this fallback was taken is also recorded. ### `just` options `--dump-graph` and `--dump-plain-graph` are cumulative From version 1.6 onwards, the options `--dump-graph` and `--dump-plain-graph` are no longer "latest wins" but cumulative. That is, if these options are given several times then `just` also writes the graph file to several destinations. In this way, it is possible to have an invocation-specific logging of the action graph without interfering with other graph logging. ### `just-mr` to support passing unique log options on each invocation The configuration file for `just-mr` has an entry `"invocation log"`. This entry, if given, is a JSON object; rc-file merging is done on the individual entries of the `"invocation log"` object. It supports the following keys. - `"directory"` A single location object specifying the directory under which the invocation logging will happen. If not given, no invocation logging will happen and the other keys are ignored. - `"project id"` A path fragment (i.e., a non-empty string, different from `"."` and `".."`, without `/` or null character). If not given, `unknown` will be assumed. - `"--profile"`, `"--dump-graph"`, `"--dump-plain-graph"`. Each a path fragment specifying the file name for the profile file, the graph file, or plain-graph file, respectively. If not given, the respective file will not be requested in the invocation of `just`. - `"meta data"` A path fragment specifying the file name of the meta-data file. If not give, no meta-data file will be created. If invocation logging is requested, `just-mr` will create for each invocation a directory `//-` where - `` is the directory specified by the `"directory"` location object. - `` is the value specified by the `"project id"` field (or `unknown`). The reason that subdirectory is specified separately is to allow projects to set their project id in the committed code base (via rc-file merging) whereas the general logging location can be specified in a user-specific or machine-specific way. - `` is the UTC timestamp (year, moth, day, hours, minutes, seconds) of the invocation and `` is a universally unique id for that invocation (following RFC9562). The reason we prefix the UUID with a time stamp is allow a simple time-based collection and clean up of the log data. Inside this directory the requested files with the specified file names will be created, in the case of `"--profile"`, `"--dump-graph"`, and `"--dump-plain-graph"` by passing the appropriate options to the invocation of `just`. The meta-data will be written just by `just-mr` itself, just before doing the `exec` call. The file contains a JSON object with keys - `"time"` The time stamp in seconds since the epoch as floating-point number, possibly rounded down to the nearest integer. - `"cmdline"` The command line, as vector of strings, that `just-mr` is about to `exec` to. - `"configuration"` The blob-identifier of the multi-repository configuration; in this way, the actual repository configuration can be conveniently backed up. In this way, analysis with respect to the source tree is possible. just-buildsystem-justbuild-b1fb5fa/doc/concepts/rules.md000066400000000000000000000576341516554100600236750ustar00rootroot00000000000000User-defined Rules ================== Targets are defined in terms of high-level concepts like "libraries", "binaries", etc. In order to translate these high-level definitions into actionable tasks, the user defines rules, explaining at a single point how all targets of a given type are built. Rules files ----------- Rules are defined in rules files (by default named `RULES`). Those contain a JSON object mapping rule names to their rule definition. For rules, the same naming scheme as for targets applies. However, built-in rules (always named by a single string) take precedence in naming; to explicitly refer to a rule defined in the current module, the module has to be specified, possibly by a relative path, e.g., `["./", ".", "install"]`. Basic components of a rule -------------------------- A rule is defined through a JSON object with various keys. The only mandatory key is `"expression"` containing the defining expression of the rule. ### `"config_fields"`, `"string_fields"` and `"target_fields"` These keys specify the fields that a target defined by that rule can have. In particular, those have to be disjoint lists of strings. For `"config_fields"` and `"string_fields"` the respective field has to evaluate to a list of strings, whereas `"target_fields"` have to evaluate to a list of target references. Those references are evaluated immediately, and in the name context of the target they occur in. The difference between `"config_fields"` and `"string_fields"` is that `"config_fields"` are evaluated before the target fields and hence can be used by the rule to specify config transitions for the target fields. `"string_fields"` on the other hand are evaluated *after* the target fields; hence the rule cannot use them to specify a configuration transition, however the target definition in those fields may use the `"outs"` and `"runfiles"` functions to have access to the names of the artifacts or runfiles of a target specified in one of the target fields. ### `"implicit"` This key specifies a map of implicit dependencies. The keys of the map are additional target fields, the values are the fixed list of targets for those fields. If a short-form name of a target is used (e.g., only a string instead of a module-target pair), it is interpreted relative to the repository and module the rule is defined in, not the one the rule is used in. Other than this, those fields are evaluated the same way as target fields settable on invocation of the rule. ### `"config_vars"` This is a list of strings specifying which parts of the configuration the rule uses. The defining expression of the rule is evaluated in an environment that is the configuration restricted to those variables; if one of those variables is not specified in the configuration the value in the restriction is `null`. ### `"config_transitions"` This key specifies a map of (some of) the target fields (whether declared as `"target_fields"` or as `"implicit"`) to a configuration expression. Here, a configuration expression is any expression in our language. It has access to the `"config_vars"` and the `"config_fields"` and has to evaluate to a list of maps. Each map specifies a transition to the current configuration by amending it on the domain of that map to the given value. ### `"imports"` This specifies a map of expressions that can later be used by `CALL_EXPRESSION`. In this way, duplication of (rule) code can be avoided. For each key, we have to have a name of an expression; expressions are named following the same naming scheme as targets and rules. The names are resolved in the context of the rule. Expressions themselves are defined in expression files, the default name being `EXPRESSIONS`. Each expression is a JSON object. The only mandatory key is `"expression"` which has to be an expression in our language. It optionally can have a key `"vars"` where the value has to be a list of strings (and the default is the empty list). Additionally, it can have another optional key `"imports"` following the same scheme as the `"imports"` key of a rule; in the `"imports"` key of an expression, names are resolved in the context of that expression. It is a requirement that the `"imports"` graph be cycle free. ### `"expression"` This specifies the defining expression of the rule. The value has to be an expression of our expression language (basically, an abstract syntax tree serialized as JSON). It has access to the following extra functions and, when evaluated, has to return a result value. #### `FIELD` The field function takes one argument, `name` which has to evaluate to the name of a field. For string fields, the given list of strings is returned; for target fields, the list of abstract names for the given target is returned. These abstract names are opaque within the rule language (but meaningful when reported in error messages) and should only be used to be passed on to other functions that expect names as inputs. #### `DEP_ARTIFACTS` and `DEP_RUNFILES` These functions give access to the artifacts, or runfiles, respectively, of one of the targets depended upon. It takes two (evaluated) arguments, the mandatory `"dep"` and the optional `"transition"`. The argument `"dep"` has to evaluate to an abstract name (as can be obtained from the `FIELD` function) of some target specified in one of the target fields. The `"transition"` argument has to evaluate to a configuration transition (i.e., a map) and the empty transition is taken as default. It is an error to request a target-transition pair for a target that was not requested in the given transition through one of the target fields. #### `DEP_PROVIDES` This function gives access to a particular entry of the provides map of one of the targets depended upon. The arguments `"dep"` and `"transition"` are as for `DEP_ARTIFACTS`; additionally, there is the mandatory argument `"provider"` which has to evaluate to a string. The function returns the value of the provides map of the target at the given provider. If the key is not in the provides map (or the value at that key is `null`), the optional argument `"default"` is evaluated and returned. The default for `"default"` is the empty list. #### `BLOB` The `BLOB` function takes a single (evaluated) argument `data` which is optional and defaults to the empty string. This argument has to evaluate to a string. The function returns an artifact that is a non-executable file with the given string as content. #### `TREE` The `TREE` function takes a single (evaluated) argument `$1` which has to be a map of artifacts. The result is a single tree artifact formed from the input map. It is an error if the map cannot be transformed into a tree (e.g., due to staging conflicts). ### `TREE_OVERLAY` and `DISJOINT_TREE_OVERLAY` The `TREE_OVERLAY` and `DISJOINT_TREE_OVERLAY` functions take a single (evaluated) argument `$1` which has to be a list of maps of artifacts. Each entry in the list is transformed into a single tree artifact formed from that map and it is an error if the map cannot be transformed into a tree (e.g., due to a staging conflict). The result is a single tree artifact defined as the overlay of the specified trees in that order (latest wins on each path after build evaluation); in the case of `DISJOINT_TREE_OVERLAY` it is a build error if a non-trivial conflict arises when computing the overlay in the build phase. #### `ACTION` Actions are a way to define new artifacts from (zero or more) already defined artifacts by running a command, typically a compiler, linker, archiver, etc. The action function takes the following arguments. - `"inputs"` A map of artifacts. These artifacts are present when the command is executed; the keys of the map are the relative path from the working directory of the command. The command must not make any assumption about the location of the working directory in the file system (and instead should refer to files by path relative to the working directory). Moreover, the command must not modify the input files in any way. (In-place operations can be simulated by staging, as is shown in the example later in this document.) It is an additional requirement that no conflicts occur when interpreting the keys as paths. For example, `"foo.txt"` and `"./foo.txt"` are different as strings and hence legitimately can be assigned different values in a map. When interpreted as a path, however, they name the same path; so, if the `"inputs"` map contains both those keys, the corresponding values have to be equal. - `"cmd"` The command to execute, given as `argv` vector, i.e., a non-empty list of strings. The 0'th element of that list will also be the program to be executed. - `"cwd"` The directory inside the action root to change to before executing the command. The directory has to be given as a string describing a non-upwards relative path. This field is optional and defaults to `""`. - `"env"` The environment in which the command should be executed, given as a map of strings to strings. - `"outs"` and `"out_dirs"` Two list of strings naming the files and directories, respectively, the command is expected to create. Those paths are interpreted relative to the action root (not relative to `"cwd"`). It is an error if the command fails to create the promised output files. These two lists have to be disjoint, but an entry of `"outs"` may well name a location inside one of the `"out_dirs"`. This function returns a map with keys the strings mentioned in `"outs"` and `"out_dirs"`. As values this map has artifacts defined to be the ones created by running the given command (in the given environment with the given inputs). #### `RESULT` The `RESULT` function is the only way to obtain a result value. It takes three (evaluated) arguments, `"artifacts"`, `"runfiles"`, and `"provides"`, all of which are optional and default to the empty map. It defines the result of a target that has the given artifacts, runfiles, and provided data, respectively. In particular, `"artifacts"` and `"runfiles"` have to be maps to artifacts, and `"provides"` has to be a map. Moreover, the keys in `"runfiles"` and `"artifacts"` are treated as paths; it is an error if this interpretation yields to conflicts. The keys in the artifacts or runfile maps as seen by other targets are the normalized paths of the keys given. Result values themselves are opaque in our expression language and cannot be deconstructed in any way. Their only purpose is to be the result of the evaluation of the defining expression of a target. #### `CALL_EXPRESSION` This function takes one mandatory argument `"name"` which is unevaluated; it has to a be a string literal. The expression imported by that name through the imports field is evaluated in the current environment restricted to the variables of that expression. The result of that evaluation is the result of the `CALL_EXPRESSION` statement. During the evaluation of an expression, rule fields can still be accessed through the functions `FIELD`, `DEP_ARTIFACTS`, etc. In particular, even an expression with no variables (that, hence, is always evaluated in the empty environment) can carry out non-trivial computations and be non-constant. The special functions `BLOB`, `ACTION`, and `RESULT` are also available. If inside the evaluation of an expression the function `CALL_EXPRESSION` is used, the name argument refers to the `"imports"` map of that expression. So the call graph is deliberately recursion free. Evaluation of a target ---------------------- A target defined by a user-defined rule is evaluated in the following way. - First, the config fields are evaluated. - Then, the target-fields are evaluated. This happens for each field as follows. - The configuration transition for this field is evaluated and the transitioned configurations determined. - The argument expression for this field is evaluated. The result is interpreted as a list of target names. Each of those targets is analyzed in all the specified configurations. - The string fields are evaluated. If the expression for a string field queries a target (via `outs` or `runfiles`), the value for that target is returned in the first configuration. The rational here is that such generator expressions are intended to refer to the corresponding target in its "main" configuration; they are hardly used anyway for fields branching their targets over many configurations. - The effective configuration for the target is determined. The target effectively has used of the configuration the variables used by the `arguments_config` in the rule invocation, the `config_vars` the rule specified, and the parts of the configuration used by a target dependent upon. For a target dependent upon, all parts it used of its configuration are relevant expect for those fixed by the configuration transition. - The rule expression is evaluated and the result of that evaluation is the result of the rule. Example of developing a rule ---------------------------- Let's consider step by step an example of writing a rule. Say we want to write a rule that programmatically patches some files. ### Framework: The minimal rule Every rule has to have a defining expression evaluating to a `RESULT`. So the minimally correct rule is the `"null"` rule in the following example rule file. { "null": {"expression": {"type": "RESULT"}}} This rule accepts no parameters, and has the empty map as artifacts, runfiles, and provided data. So it is not very useful. ### String inputs Let's allow the target definition to have some fields. The most simple fields are `string_fields`; they are given by a list of strings. In the defining expression we can access them directly via the `FIELD` function. Strings can be used when defining maps, but we can also create artifacts from them, using the `BLOB` function. To create a map, we can use the `singleton_map` function. We define values step by step, using the `let*` construct. ``` jsonc { "script only": { "string_fields": ["script"] , "expression": { "type": "let*" , "bindings": [ [ "script content" , { "type": "join" , "separator": "\n" , "$1": { "type": "++" , "$1": [["H"], {"type": "FIELD", "name": "script"}, ["w", "q", ""]] } } ] , [ "script" , { "type": "singleton_map" , "key": "script.ed" , "value": {"type": "BLOB", "data": {"type": "var", "name": "script content"}} } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "script"}} } } } ``` ### Target inputs and derived artifacts Now it is time to add the input files. Source files are targets like any other target (and happen to contain precisely one artifact). So we add a target field `"srcs"` for the file to be patched. Here we have to keep in mind that, on the one hand, target fields accept a list of targets and, on the other hand, the artifacts of a target are a whole map. We chose to patch all the artifacts of all given `"srcs"` targets. We can iterate over lists with `foreach` and maps with `foreach_map`. Next, we have to keep in mind that targets may place their artifacts at arbitrary logical locations. For us that means that first we have to make a decision at which logical locations we want to place the output artifacts. As one thinks of patching as an in-place operation, we chose to logically place the outputs where the inputs have been. Of course, we do not modify the input files in any way; after all, we have to define a mathematical function computing the output artifacts, not a collection of side effects. With that choice of logical artifact placement, we have to decide what to do if two (or more) input targets place their artifacts at logically the same location. We could simply take a "latest wins" semantics (keep in mind that target fields give a list of targets, not a set) as provided by the `map_union` function. We chose to consider it a user error if targets with conflicting artifacts are specified. This is provided by the `disjoint_map_union` that also allows to specify an error message to be provided the user. Here, conflict means that values for the same map position are defined in a different way. The actual patching is done by an `ACTION`. We have the script already; to make things easy, we stage the input to a fixed place and also expect a fixed output location. Then the actual command is a simple shell script. The only thing we have to keep in mind is that we want useful output precisely if the action fails. Also note that, while we define our actions sequentially, they will be executed in parallel, as none of them depends on the output of another one of them. ``` jsonc { "ed patch": { "string_fields": ["script"] , "target_fields": ["srcs"] , "expression": { "type": "let*" , "bindings": [ [ "script content" , { "type": "join" , "separator": "\n" , "$1": { "type": "++" , "$1": [["H"], {"type": "FIELD", "name": "script"}, ["w", "q", ""]] } } ] , [ "script" , { "type": "singleton_map" , "key": "script.ed" , "value": {"type": "BLOB", "data": {"type": "var", "name": "script content"}} } ] , [ "patched files per target" , { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": "srcs"} , "body": { "type": "foreach_map" , "var_key": "file_name" , "var_val": "file" , "range": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "src"}} , "body": { "type": "let*" , "bindings": [ [ "action output" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "script"} , { "type": "singleton_map" , "key": "in" , "value": {"type": "var", "name": "file"} } ] } , "cmd": [ "/bin/sh" , "-c" , "cp in out && chmod 644 out && /bin/ed out < script.ed > log 2>&1 || (cat log && exit 1)" ] , "outs": ["out"] } ] ] , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "file_name"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "action output"} , "key": "out" } } } } } ] , [ "artifacts" , { "type": "disjoint_map_union" , "msg": "srcs artifacts must not overlap" , "$1": { "type": "++" , "$1": {"type": "var", "name": "patched files per target"} } } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "artifacts"}} } } } ``` A typical invocation of that rule would be a target file like the following. ``` jsonc { "input.txt": { "type": "ed patch" , "script": ["%g/world/s//user/g", "%g/World/s//USER/g"] , "srcs": [["FILE", null, "input.txt"]] } } ``` As the input file has the same name as a target (in the same module), we use the explicit file reference in the specification of the sources. ### Implicit dependencies and config transitions Say, instead of patching a file, we want to generate source files from some high-level description using our actively developed code generator. Then we have to do some additional considerations. - First of all, every target defined by this rule not only depends on the targets the user specifies. Additionally, our code generator is also an implicit dependency. And as it is under active development, we certainly do not want it to be taken from the ambient build environment (as we did in the previous example with `ed` which, however, is a pretty stable tool). So we use an `implicit` target for this. - Next, we notice that our code generator is used during the build. In particular, we want that tool (written in some compiled language) to be built for the platform we run our actions on, not the target platform we build our final binaries for. Therefore, we have to use a configuration transition. - As our defining expression also needs the configuration transition to access the artifacts of that implicit target, we better define it as a reusable expression. Other rules in our rule collection might also have the same task; so `["transitions", "for host"]` might be a good place to define it. In fact, it can look like the expression with that name in our own code base. So, the overall organization of our rule might be as follows. ``` jsonc { "generated code": { "target_fields": ["srcs"] , "implicit": {"generator": [["generators", "foogen"]]} , "config_vars": ["HOST_ARCH"] , "imports": {"for host": ["transitions", "for host"]} , "config_transitions": {"generator": [{"type": "CALL_EXPRESSION", "name": "for host"}]} , "expression": ... } } ``` ### Providing information to consuming targets In the simple case of patching, the resulting file is indeed the only information the consumer of that target needs; in fact, the main point was that the resulting target could be a drop-in replacement of a source file. A typical rule, however, defines something like a library and a library is much more, than just the actual library file and the public headers: a library may depend on other libraries; therefore, in order to use it, we need - to have the header files of dependencies available that might be included by the public header files of that library, - to have the libraries transitively depended upon available during linking, and - to know the order in which to link the dependencies (as they might have dependencies among each other). In order to keep a maintainable build description, all this should be taken care of by simply depending on that library. We do *not* want the consumer of a target having to be aware of such transitive dependencies (e.g., when constructing the link command line), as it used to be the case in early build tools like `make`. It is a deliberate design choice that a target is given only by the result of its analysis, regardless of where it is coming from. Therefore, all this information needs to be part of the result of a target. Such kind of information is precisely, what the mentioned `"provides"` map is for. As a map, it can contain an arbitrary amount of information and the interface function `"DEP_PROVIDES"` is in such a way that adding more providers does not affect targets not aware of them (there is no function asking for all providers of a target). The keys and their meaning have to be agreed upon by a target and its consumers. As the latter, however, typically are a target of the same family (authored by the same group), this usually is not a problem. A typical example of computing a provided value is the `"link-args"` in the rules used by `just` itself. They are defined by the following expression. ``` jsonc { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "keys", "$1": {"type": "var", "name": "lib"}} , {"type": "CALL_EXPRESSION", "name": "link-args-deps"} , {"type": "var", "name": "link external", "default": []} ] } } ``` This expression - collects the respective provider of its dependencies, - adds itself in front, and - deduplicates the resulting list, keeping only the right-most occurrence of each entry. In this way, the invariant is kept, that the `"link-args"` from a topological ordering of the dependencies (in the order that a each entry is mentioned before its dependencies). just-buildsystem-justbuild-b1fb5fa/doc/concepts/service-target-cache.md000066400000000000000000000452571516554100600265260ustar00rootroot00000000000000Target-level caching as a service ================================= Motivation ---------- Projects can have quite a lot of dependencies that are not part of the build environment, but are, instead, built from source, e.g., in order to always build against the latest snapshot. The latter is a typical workflow in case of first-party dependencies. In the case of `justbuild`, those first-party dependencies form a separate logical repository that is typically content fixed (e.g., because that dependency is versioned in a `git` repository). Moreover, code is typically first built (and tested) by the owning project before being used as a dependency. Therefore, if remote execution is used, for a first-party dependency, we expect all actions to be in cache. As dependencies are typically updated less often than the code being developed is changed, in most builds, the dependencies are in target-level cache. In other words, in a remote-execution setup, the whole code of dependencies is fetched just to walk through the action graph a single time to get the necessary cache hits. Core concepts and implementation -------------------------------- To avoid these unnecessary fetches, we have added a new subcommand `just serve` that start a main service that provides the dependencies. This typically happens by looking up a target-level cache entry. If the entry, however, is not in cache, this also includes building the respective `export` target using an associated remote-execution endpoint. ### Scope: eligible `export` targets In order to typically have requests in cache, `just serve` will refuse to handle requests that do not refer to `export` targets in content-fixed repositories; recall that for a repository to be content fixed, so have to be all repositories reachable from there. ### Communication through an associated remote-execution service Each `just serve` endpoint is always associated with a remote-execution endpoint. All artifacts exchanged between client and `just serve` endpoint are exchanged via the CAS that is part in the associated remote-execution endpoint. This remote-execution endpoint is also used if `just serve` has to build targets. The associated remote-execution endpoint can well be the same process simultaneously acting as `just execute`. In fact, this is the default if no remote-execution endpoint is specified. ### Sources: local git repositories and remote trees A `just serve` instance takes roots from various sources, - the `git` repository contained in the local build root, - additional `git` repositories, optionally specified in the invocation, and - as last resort, asking the CAS in the designated remote-execution service for the specified `git` tree. Allowing a list of repositories to take as sources (rather than a single one) increases the effort when having to search for a specified tree (e.g., in case the requested `export` target is not in cache and an actual analysis of the build has to be carried out) or specific commit (e.g., in case a client asks for the tree of a given commit). However, it allows for the natural workflow of keeping separate upstream repositories in separate clones (updated in an appropriate way) without artificially putting them in a single repository (as orphan branches). Supporting building against trees from CAS allows more flexibility in defining roots that clients do not have to care about. In fact, they can be defined in any way, as long as - the client is aware of the git tree identifier of the root, and - some entity ensures the needed trees are known to the CAS. The auxiliary changes to `just-mr` described later in this document provide one possible way to handle archives in this way. Moreover, this additional flexibility will be necessary if we ever support computed roots, i.e., roots that are the output of a `just` build. ### Delegation: absent roots in `just` repository specification In order for `just` to know for which repositories to delegate the build to the designated `just serve` endpoint, the repository configuration for `just` can mark roots as _absent_; this is done by only giving the type as `"git tree"` (or the corresponding ignore-special variant thereof) and the tree identifier in the root specification, but no witnessing repository. Any repository containing an absent root has to be content fixed, but not all roots have to be absent (as `just` can always upload those trees to CAS). It is an error if, outside the computations delegated to `just serve`, a non-export target is requested from a repository containing an absent root. Moreover, whenever there is a dependency on a repository containing an absent root, a `just serve` endpoint has to be specified in the invocation of `just`. Protocol description -------------------- Communication is handled via `grpc` exchanging `proto` buffers containing the information described in the rest of this section. Besides the main service of `just serve`, auxiliary requests are defined, bundled in two other services: one allowing `just-mr` to configure multi-repository builds in the context of `absent` roots, and the other to perform the optional check for remote-execution endpoint consistency between a client and the `just serve` endpoint. ### Main service #### Main request and answer format A request is given by - the map of remote-execution properties for the designated remote-execution endpoint, - the identifier of the blob containing the endpoint configuration information; together with the knowledge on the fixed endpoint, the `just serve` instance computes the target-level cache shard, and - the identifier of the target-level cache key; it is the client's responsibility to ensure that the referred blob (i.e., the JSON object with appropriate values for the keys `"repo_key"`, `"target_name"`, and `"effective_config"`) as well as the indirectly referred repository description (the JSON object the `"repo_key"` in the cache key refers to) are uploaded to CAS (of the designated remote-execution endpoint) beforehand. The answer to that request is the identifier of the corresponding target-level cache value (in the same format as for local target-level caching). The `just serve` instance ensures that the actual value, as well as any directly or indirectly referenced artifacts are available in the respective remote-execution CAS. Alternatively, the answer indicates the kind of error (unknown root, not an export target, build failure, etc). #### Auxiliary request: flexible variables of an `export` target To allow `just` to compute the target-level cache key without knowledge of an absent tree, `just serve` also answers questions about the flexible variables of an `export` target. Such an `export` target is specified by the tree of its target-level root, the name of the targets file, and the name of the target itself. The answer is a list of strings, naming the flexible variables. #### Auxiliary request: rule description of an `export` target To support `just describe` also in the cases where code is delegated to the `just serve` endpoint, an additional request for the `describe` information of a target can be requested; as `just serve` only handles `export` targets, this target necessarily has to be an export target. The request again contains the tree identifier of the target-level root, the name of the targets file, and the name of the target to inspect. The answer is the identifier of a blob containing a JSON object with the needed information, i.e., those parts of the target description that are used by `just describe`. Alternatively, the answer indicates the kind of error (unknown root, not an export target, etc). ### Auxiliary service: source trees #### Auxiliary request: tree of a commit For `git` repositories it is common to specify a commit in order to fix a dependency (even though the corresponding tree identifier would be enough). Moreover, the standard `git` protocol supports asking for the commit of a given remote branch, but additional overhead is needed in order to get the tree identifier. Therefore, in order to support clients (or, more precisely, `just-mr` instances setting up the repository description) in constructing an appropriate request for `just serve` without unnecessary overhead, `just serve` supports a second kind of request, where the client request consists of a `git` commit identifier and the server answers with the tree identifier for that commit if it is aware of that commit, or indicates that it is not aware of that commit. Optionally, the client can request that `just serve` back up this tree in the CAS of the associated remote-execution endpoint. #### Auxiliary request: tree of an archive For archives typically the `git` blob identifier is given, rather than the tree. In order to allow `just-mr` to set up a repository description without fetching the respective archive, `just serve` supports also a request which, given the blob identifier of an archive, answers with the respective tree identifier of the unpacked archive. Here, if `just serve` needs the archive, it can look it up in its CAS, any of the supplied `git` repositories (where one might be for archiving of the third-party distribution archives), and the specified remote-execution endpoint. The (functional!) association of archive blob identifier to tree identifier of the unpacked archive is stored in the local build root and the respective tree is fixed in the `git` repository of the local build root in the same way as `just-mr` does it. When answering such a request, that tree map is consulted first (so that those requests as well can be typically served from cache). Optionally, the client can request that `just serve` back up this tree in the CAS of the associated remote-execution endpoint. #### Auxiliary request: tree of a distdir For archives we typically require the `git` blob identifier to be given, thus the tree identifier corresponding to a distdir (i.e., a list of distfiles) can always be computed without fetching the actual archives. In order to allow `just-mr` to set up a repository description that can build against an _absent_ distdir repository root, `just serve` supports a request which, given a mapping of distfile names to their content blob identifiers, returns the tree identifier of a directory containing that list of distfiles, with the guarantee that all content blobs are known to `just serve`. The locations they are looked for are, in order: the local CAS, all known Git repositories (including the local Git cache), and the CAS of the associated remote-execution endpoint. Any blob located in a Git repository is made available in the local CAS. Optionally, the client can request that `just serve` back up this tree and all the content blobs in the CAS of the associated remote-execution endpoint. #### Auxiliary requests: known Git objects For `just fetch` operations typically either a blob (e.g., content of an archive) or a tree (e.g., a root, like from a `git tree` repository) are needed to be stored into local CAS. For these cases, two auxiliary requests, one for blobs and one for trees, respectively, have been provided. They check whether the `just serve` endpoint knows these Git objects and, if yes, ensure they are uploaded to the remote CAS, from where the client can easily then retrieve them. #### Auxiliary requests: check and get trees In `just-mr`, the `to_git` pragma most often is used to make sure a local root is available in a content-defined manner as a Git-tree. This allows it to be used in the build description of export targets. It would be beneficial then for export targets built by a serve endpoint to have access to such roots, which often describe exactly the target(s) we want built. In order to facilitate this, two more auxiliary requests are added. The first request supported, given a tree identifier corresponding to a root, checks whether the serve endpoint has _direct_ access to it, meaning it is available to it locally, and, if found, it makes sure the tree is made available in a location where the serve endpoint can build against it. The places checked for this tree are, in order: the Git cache, the known Git repositories, and the local CAS. The second request supported, given a tree identifier, retrieves a tree from the CAS of the associated remote-execution endpoint and makes it available in a location where the serve endpoint can build against it, with the understanding that the client has ensured beforehand that the tree exists in the remote CAS. This is because, in the typical use case, a client will first perform a check via the first request above, and if the serve endpoint reports that it doesn't know the tree, the client can upload it to the remote CAS and ask the serve endpoint to retrieve it via the second request. This ensures that a client can avoid unnecessary uploads to the remote CAS, while making sure that the serve endpoint has the roots marked absent available to build against. ### Auxiliary service: configuration #### Auxiliary request: remote-execution endpoint Given that all artifact exchanges between client and `just serve` rely on the CAS of a given remote endpoint, the client might want to double check that the remote execution endpoint it wants to use is the same that is associated with the `just serve` instance. The server replies with the address (in the usual `HOST:PORT` string format) of the associated remote execution endpoint, if set, or an empty string otherwise (i.e., if the serve endpoint acts also as execution endpoint). Auxiliary changes ----------------- ### Modifications to the justbuild analysis of an export target During the analysis of an export target, querying the `just serve` endpoint is exclusively linked to the presence of at least one _absent_ root. The first time that we need to query `just serve` we verify that its remote endpoint coincides with the one given to `just`. If the _target root_ for this export target is marked as absent: - We query the `just serve` for retrieving the flexible configuration variables needed to compute the target cache key. If `just serve` cannot answer, we break the analysis and inform the user with a proper error message. - With the served flexible configuration variables we compute the target cache key, as all other required information for this in available locally. If the cache entry is not in the local target cache, we query `just serve` to provide the associated target cache value. If it is not able to provide the target cache value, analysis fails and we error out. It has to be noted that, in the case the `just serve` endpoint also does not have the target cache entry in its own target cache, a build of the content-fixed target is dispatched to the associated remote-execution endpoint, which will thus increase the time spent in the analysis phase, as experienced by the user. In order to provide a better user experience, the work done by the `just serve` endpoint is also being reported to the end user, similarly to the reporting done for a locally-triggered build. #### `just-mr` pragma `"absent"` For `just-mr` to know how to construct the multi-repository description, the description used by `just-mr` was extended. More precisely, a new key `"absent"` is allowed in the `"pragma"` dictionary of a repository description. If the specified value is true, `just-mr` generates an absent root out of this description, using all available means to generate that root without ever having to fetch the repository locally. For example, in the typical case of a `git` repository the auxiliary `just serve` function to obtain the tree of a commit is used. To allow this communication, `just-mr` also accepts arguments describing a `just serve` endpoint and forwards them as early arguments to `just`, in the same way as it does, e.g., with `--local-build-root`. If a `just serve` endpoint is given to `just-mr`, the tool ensures however possible that all absent roots it generates are available also to the serve endpoint for a subsequent orchestrated remote build. Absent roots without providing a serve endpoint can also be generated, however this is not a typical use case and the tool provides warnings in this regard. #### `just-mr` to inquire remote execution before fetching In line with the idea that fetching sources from upstream should happen only once and not once per developer, we have added remote execution as another way of obtaining files to `just-mr`. More precisely, `just-mr` now supports the options `just` accepts to connect to the remote CAS. When given, those are forwarded to `just` as early arguments (so that later `just`-only ones can override them); moreover, when a file needed to set up a (present) root is found neither in local CAS nor in one of the specified distdirs, `just-mr` first asks the remote CAS for the missing file before trying to fetch itself from the specified URL. The rationale for this search order is that the designated remote-execution service is typically reachable over the network in a more reliable way than external resources (while local resources do not require a network at all). #### `just-mr` to support new repository type `git tree` A new repository type is added to `just-mr`, called `git tree`. Such a repository is given by - a `git` tree identifier, and - a command that, when executed in an empty directory (anywhere in the file system) will create in that directory a directory structure containing the specified `git` tree (either top-level or in some subdirectory). Moreover, that command does not modify anything outside the directory it is called in; it is an error if the specified tree is not created in this way. In this way, content-fixed repositories can be generated in a generic way, e.g., using other version-control systems or specialized artifact-fetching tools. #### `just-mr fetch` to support storing in remote-execution CAS The `fetch` subcommand of `just-mr` will get an additional option to support backing up the fetched information not to a local directory, but instead to the CAS of the specified remote-execution endpoint. This includes - all archives fetched, but also - all trees computed in setting up the respective repository description, both, from `git tree` repositories, as well as from archives. In this way, `just-mr` can be used to fill the CAS from one central point with all the information the clients need to treat all content-fixed roots as absent. ### Target-level cache writing in the presence of some targets served When building, `just` normally does not create an entry for target-level cache hit received from `just serve`. However, it might happen that `just` has to analyse an eligible `export` target locally, as the `just serve` instance cannot provide it, and during that analysis `export` targets provided by `just serve` are encountered. In this case, the writing of the export targets depending on served targets is skipped. just-buildsystem-justbuild-b1fb5fa/doc/concepts/symlinks.md000066400000000000000000000153401516554100600244000ustar00rootroot00000000000000Symbolic links ============== Background ---------- Besides files and directories, symbolic links are also an important entity in the file system. Also `git` natively supports symbolic links as entries in a tree object. Technically, a symbolic link is a string that can be read via `readlink(2)`. However, they can also be followed and functions to access a file, like `open(2)` do so by default. When following a symbolic link, both, relative and absolute, names can be used. Symbolic links in build systems ------------------------------- ### Follow and reading both happen Compilers usually follow symlinks for all inputs. Archivers (like `tar(1)` and package-building tools) usually read the link in order to package the link itself, rather than the file referred to (if any). As a generic build system, it is desirable to not have to make assumptions on the intention of the program called (and hence the way it deals with symlinks). This, however, has the consequence that only symbolic links themselves can properly model symbolic links. ### Self-containedness and location-independence of roots From a build-system perspective, a root should be self-contained; in fact, the target-level caching assumes that the git tree identifier entirely describes a `git`-tree root. For this to be true, such a root has to be both, self contained and independent of its (assumed) location in the file system. In particular, we can neither allow absolute symbolic links (as they, depending on the assumed location, might point out of the root), nor relative symbolic links that go upwards (via a `../` reference) too far. ### Symbolic links in actions Like for source roots, we understand action directories as self contained and independent of their location in the file system. Therefore, we have to require the same restrictions there as well, i.e., neither absolute symbolic links nor relative symbolic links going up too far. Allowing all relative symbolic links that don't point outside the action directory, however, poses an additional layer of complications in the definition of actions: a string might be allowed as symlink in some places in the action directory, but not in others; in particular, we can't tell only from the information that an artifact is a relative symlink whether it can be safely placed at a particular location in an action or not. Similarly for trees for which we only know that they might contain relative symbolic links. ### Presence of symbolic links in system source trees It can be desirable to use system libraries or tools as dependencies. A typical use case, but not the only one, is packaging a tool for a distribution. An obvious approach is to declare a system directory as a root of a repository (providing the needed target files in a separate root). As it turns out, however, those system directories do contain symbolic links, e.g., shared libraries pointing to the specific version (like `libfoo.so.3` as a symlink pointing to `libfoo.so.3.1.4`) or detours through `/etc/alternatives`. Bootstrapping "shopping list" option ------------------------------------ In order to more easily support building the tool itself against pre-installed dependencies with the respective directories containing symbolic links, or tools (like `protoc`) being symbolic links (e.g., to the specific version), repositories can specify, in the `"copy"` attribute of the `"local_bootstrap"` parameter, a list of files and directories to be copied as part of the bootstrapping process to a fresh clean directory serving as root; during this copying, symlinks are followed. Our treatment of symbolic links ------------------------------- ### "Ignore-special" roots For cases where we simply have no need for special entries, all the existing roots have "ignore-special" versions thereof. In such a root (regardless whether file based, or `git`-tree based), everything not a file or a directory is pretended to be absent. For any compile-like tasks, the effect of symlinks can be modeled by appropriate staging. As certain entries have to be ignored, source trees can only be obtained by traversing the respective tree; in particular, the `TREE` reference is no longer constant time on those roots, even if `git`-tree based. Nevertheless, for `git`-tree roots, the effective tree is a function of the `git`-tree of the root, so `git`-tree-based ignore-special roots are content fixed and hence eligible for target-level caching. ### Non-upwards relative symlinks as first-class objects A restricted form of symlinks, more precisely *relative* *non-upwards symbolic links*, exist as first-class objects. That is, a new artifact type (besides blobs and trees) for relative non-upwards symbolic links has been introduced. Like any other artifact, they can be freely placed into the inputs of an action, as well as in artifacts, runfiles, or provides map of a target. Artifacts of this new type can be defined as: - source-symlink reference, as well as implicitly as part of a source tree, - as a symlink output of an action, as well as implicitly as part of a tree output of an action, and - explicitly in the rule language from a string through a new `SYMLINK` constructor function. While existing as separate artifacts in order to properly stage them, (relative non-upwards) symbolic links are, in many aspects, simple files with elevated permissions. As such, they locally use the existing file CAS. Remotely, the existing execution protocol already allows the handling of symbolic links via corresponding Protobuf messages, therefore no extensions are needed. Additionally, the built-in rules are extended with a `"symlink"` target, allowing the generation of a symlink with given non-upwards target path. ### Import resolved `git`-trees Finally, to be as flexible as possible in handling external repositories with (possibly) upwards symbolic links, we allow filesystem directories and archives to be imported also as partially or completely resolved `git`-trees. In a *partially resolved tree*, all relative upwards symbolic links confined to the tree get resolved, i.e., replaced by a copy of the entry they point to, if existing, or removed otherwise. This of course leaves relative non-upwards symbolic links in the `git`-tree, as they are supported objects. Alternatively, in a *completely resolved tree*, all relative symbolic links confined to the tree (whether upwards or not) get resolved, resulting in a `git`-tree free of all symbolic links. For reasons already described, absolute symbolic links are never supported. As this process acts directly at the repository level, the resulting roots remain cacheable and their trees accessible in constant time. Moreover, to increase the chances of cache hits in `just-mr`, not only the resulting resolved trees are stored, but also the original, unresolved ones. just-buildsystem-justbuild-b1fb5fa/doc/concepts/target-cache.md000066400000000000000000000342061516554100600250600ustar00rootroot00000000000000Target-level caching ==================== `git` trees as content-fixed roots ---------------------------------- ### The `"git tree"` root scheme The multi-repository configuration supports a scheme `"git tree"`. This scheme is given by two parameters, - the id of the tree (as a string with the hex encoding), and - an arbitrary `git` repository containing the specified tree object, as well as all needed tree and blob objects reachable from that tree. For example, a root could be specified as follows. ``` jsonc ["git tree", "6a1820e78f61aee6b8f3677f150f4559b6ba77a4", "/usr/local/src/justbuild.git"] ``` It should be noted that the `git` tree identifier alone already specifies the content of the full tree. However, `just` needs access to some repository containing the tree in order to know what the tree looks like. Nevertheless, it is an important observation that the tree identifier alone already specifies the content of the whole (logical) directory. The equality of two such directories can be established by comparing the two identifiers *without* the need to read any file from disk. Those "fixed-content" descriptions, i.e., descriptions of a repository root that already fully determines the content are the key to caching whole targets. ### `KNOWN` artifacts The in-memory representation of known artifacts has an optional reference to a repository containing that artifact. Artifacts "known" from local repositories might not be known to the CAS used for the action execution; this additional reference allows to fill such misses in the CAS. Content-fixed repositories -------------------------- ### The parts of a content-fixed repository In order to meaningfully cache a target, we need to be able to efficiently compute the cache key. We restrict this to the case where we can compute the information about the repository without file-system access. This requires that all roots (workspace, target root, etc) be content fixed, as well as the bindings of the free repository names (and hence also all transitively reachable repositories). The call such repositories "content-fixed" repositories. ### Canonical description of a content-fixed repository The local data of a repository consists of the following. - The roots (for workspace, targets, rules, expressions). As the tree identifier already defines the content, we leave out the path to the repository containing the tree. - The names of the targets, rules, and expression files. - The names of the outgoing "bindings". Additionally, repositories can reach additional repositories via bindings. Moreover, this repository-level dependency relation is not necessarily cycle free. In particular, we cannot use the tree unfolding as canonical representation of that graph up to bisimulation, as we do with most other data structures. To still get a canonical representation, we factor out the largest bisimulation, i.e., minimize the respective automaton (with repositories as states, local data as locally observable properties, and the binding relation as edges). Finally, for each repository individually, the reachable repositories are renamed `"0"`, `"1"`, `"2"`, etc, following a depth-first traversal starting from the repository in question where outgoing edges are traversed in lexicographical order. The entry point is hence recognisable as repository `"0"`. The repository key content-identifier of the canonically formatted canonical serialisation of the JSON encoding of the obtain multi-repository configuration (with repository-free git-root descriptions). The serialisation itself is stored in CAS. These identifications and replacement of global names does not change the semantics, as our name data types are completely opaque to our expression language. In the `"json_encode"` expression, they're serialized as `null` and string representation is only generated in user messages not available to the language itself. Moreover, names cannot be compared for equality either, so their only observable properties, i.e., the way `"DEP_ARTIFACTS"`, `"DEP_RUNFILES`, and `"DEP_PROVIDES"` reacts to them are invariant under repository bisimulation. Configuration and the `"export"` rule ------------------------------------- Targets not only depend on the content of their repository, but also on their configurations. Normally, the effective part of a configuration is only determined after analysing the target. However, for caching, we need to compute the cache key directly. This property is provided by the built-in `"export"` rule; only `"export"` targets residing in content-fixed repositories will be cached. This also serves as indication, which targets of a repository are intended for consumption by other repositories. An `"export"` rule takes precisely the following arguments. - `"target"` specifying a single target, the target to be cached. It must not be tainted. - `"flexible_config"` a list of strings; those specify the variables of the configuration that are considered. All other parts of the configuration are ignored. So the effective configuration for the `"export"` target is the configuration restricted to those variables (filled up with `null` if the variable was not present in the original configuration). - `"fixed_config"` a dict with of arbitrary JSON values (taken unevaluated) with keys disjoint from the `"flexible_config"`. An `"export"` target is analyzed as follows. The configuration is restricted to the variables specified in the `"flexible_config"`; this will result in the effective configuration for the exported target. It is a requirement that the effective configuration contain only pure JSON values. The (necessarily conflict-free) union with the `"fixed_config"` is computed and the `"target"` is evaluated in this configuration. The result (artifacts, runfiles, provided information) is the result of that evaluation. It is a requirement that the provided information does only contain pure JSON values and artifacts (including tree artifacts); in particular, they may not contain names. Cache key --------- We only consider `"export"` targets in content-fixed repositories for caching. An export target is then fully described by - the repository key of the repository the export target resides in, - the target name of the export target within that repository, described as module-name pair, and - the effective configuration. More precisely, the canonical description is the JSON object with those values for the keys `"repo_key"`, `"target_name"`, and `"effective_config"`, respectively. The repository key is the blob identifier of the canonical serialisation (including sorted keys, etc) of the just described piece of JSON. To allow debugging and cooperation with other tools, whenever a cache key is computed, it is ensured, that the serialisation ends up in the applicable CAS. It should be noted that the cache key can be computed *without* analyzing the target referred to. This is possible, as the configuration is pruned a priori instead of the usual procedure to analyse and afterwards determine the parts of the configuration that were relevant. Cached value ------------ The value to be cached is the result of evaluating the target, that is, its artifacts, runfiles, and provided data. All artifacts inside those data structures will be described as known artifacts. As serialisation, we will essentially use our usual JSON encoding; while this can be used as is for artifacts and runfiles where we know that they have to be a map from strings to artifacts, additional information will be added for the provided data. The provided data can contain artifacts, but also legitimately pure JSON values that coincide with our JSON encoding of artifacts; the same holds true for nodes and result values. Moreover, the tree unfolding implicit in the JSON serialisation can be exponentially larger than the value. Therefore, in our serialisation, we add an entry for every subexpression and separately add a list of which subexpressions are artifacts, nodes, or results. During deserialisation, we use this subexpression structure to deserialize every subexpression only once. Sharding of target cache ------------------------ In our target description, the execution environment is not included. For local execution, it is implicit anyway. As we also want to cache high-level targets when using remote execution, we shard the target cache (e.g., by using appropriate subdirectories) by the blob identifier of the serialisation of the description of the execution backend. Here, `null` stands for local execution, and for remote execution we use an object with keys `"remote_execution_address"` and `"remote_execution_properties"` filled in the obvious way. As usual, we add the serialisation to the CAS. `"export"` targets, strictness and the extensional projection ------------------------------------------------------------- As opposed to the target that is exported, the corresponding export target, if part of a content-fixed repository, will be strict: a build depending on such a target can only succeed if all artifacts in the result of target (regardless whether direct artifacts, runfiles, or as part of the provided data) can be built, even if not all (or even none) are actually used in the build. Upon cache hit, the artifacts of an export target are the known artifacts corresponding to the artifacts of the exported target. While extensionally equal, known artifacts are defined differently, so an export target and the exported target are intensionally different (and that difference might only be visible on the second build). As intensional equality is used when testing for absence of conflicts in staging, a target and its exported version almost always conflict and hence should not be used together. One way to achieve this is to always use the export target for any target that is exported. This fits well together with the recommendation of only depending on export targets of other repositories. If a target forwards artifacts of an exported target (indirect header files, indirect link dependencies, etc), and is exported again, no additional conflicts occur; replacing by the corresponding known artifact is a projection: the known artifact corresponding to a known artifact is the artifact itself. Moreover, by the strictness property described earlier, if an export target has a cache hit, then so have all export targets it depends upon. Keep in mind that a repository can only be content-fixed if all its dependencies are. For this strictness-based approach to work, it is, however, a requirement that any artifact that is exported (typically indirectly, e.g., as part of a common dependency) by several targets is only used through the same export target. For a well-structured repository, this should not be a natural property anyway. The forwarding of artifacts are the reason we chose that in the non-cached analysis of an export target the artifacts are passed on as received and are not wrapped in an "add to cache" action. The latter choice would violate that projection property we rely upon. ### Example Consider the following target file (on a content-fixed root) as example. ``` jsonc { "generated": {"type": "generic", "outs": ["out.txt"], "cmds": ["echo Hello > out.txt"]} , "export": {"type": "export", "target": "generated"} , "use": {"type": "install", "dirs": [["generated", "."], ["generated", "other-use"]]} , "": {"type": "export", "target": "use"} } ``` Upon initial analysis (on an empty local build root) of the default target `""`, the output artifact `out.txt` is an action artifact, more precisely the same one that is output of the target `"generated"`; the target `"export"` also has the same artifact on output. After building the default target, a target-cache entry will be written for this target, containing the extensional definition of the target, so for `out.txt` the known artifact `e965047ad7c57865...` stored; as a side effect, also for the target `"export"` a target-cache entry will be written, containing, of course, the same known artifact. So on subsequent analysis, both `"export"` and `""` will still have the same artifact for `out.txt`, but this time a known one. This artifact is now different from the artifact of the target `"generated"` (which is still an action artifact), but no conflicts arise as the usual target discipline requires that any target not a (direct or indirect) dependency of `"export"` use the target `"generated"` only indirectly by using the target `"export"`. Also note that further exporting such a target has to effect, as a known artifact always evaluates to itself. In that sense, replacing by the extensional definition is a projection. ### Interaction with garbage collection While adding the implied export targets happens automatically due to the evaluation mechanism, the dependencies of target-level cache entries on one another still have to be persisted to honor them during garbage collection. Otherwise it would be possible that an implied target gets garbage collected. In fact, that would even be likely as typical builds only reference the top-level export targets. #### Analysis to track the export targets depended upon As we have to persist this dependency, we need to explicitly track it. More precisely, the internal data structure of an analyzed target is extended by a set of all the export targets eligible for caching, represented by the hashes of the `TargetCacheKey`s, encountered during the analysis of that target. ### Extension of the value of a target-level cache entry The cached value for a target-level cache entry is serialized as a JSON object, with besides the keys `"artifacts"`, `"runfiles"`, and `"provides"` also a key `"implied export targets"` that lists (in lexicographic order) the hashes of the cache keys of the export targets the analysis of the given export target depends upon; the field is only serialized if that list is non empty. ### Additional invariant honored during uplinking Our cache honors the additional invariant that, whenever a target-level cache entry is present, so are the implied target-level cache entries. This invariant is honored when adding new target-level cache entries by adding them in the correct order, as well as when uplinking by uplinking the implied entries first (and there, of course, honoring the respective invariants). just-buildsystem-justbuild-b1fb5fa/doc/concepts/tree-overlay.md000066400000000000000000000141421516554100600251440ustar00rootroot00000000000000# Tree-Overlay Actions ## Use cases for tree-overlay actions Our build tool has tree objects as first-class citizens. Trees can be obtained as directory outputs of actions, as well as by an explicit tree constructor in a rule definition taking an arrangement of artifacts and constructing a tree from it. Trees are handled as opaque objects, which has two advantages. - From a technical point, this allows passing through potentially large directories by simply passing on a single identifier. - From a user point of view, this improves maintainability, as a certain target can already claim certain subtrees in its artifacts or runfiles, so that staging conflicts that might arise from a latter addition of artifacts are already detected now. However, there are some use cases not covered by this way of handling trees. E.g., when creating disk images, it might be desirable to add project-specific artifacts to a tree obtained as directory output of an action calling a foreign build system. Of course, there need to be some out-of-band understanding where artifacts can be placed without messing up the original tree, but often this is the case, despite this being hard to formulate in a way that can be verified by a build system; however, it is easy for a build system to verify after the fact (i.e., once the trees are computed) that certain trees do not conflict on any path. Such an after-the-fact check is often maintainable enough and still guarantees that no files needed for the disk image to work get lost during the build process. A similar need for overlaying trees might occur when a third-party library is built using a foreign build system and, in order to keep the description maintainable over updates, the include files are collected as a whole directory. The mentioned use cases can be done in `justbuild` by means of the in-memory actions of type `TREE_OVERLAY` that rules can use to construct new trees out of existing ones by overlaying the contents; there is also a variant `DISJOINT_TREE_OVERLAY` that enforces that the overlaid trees do not conflict on any path. For ad-hoc constructions, there are built-in rules `tree_overlay` and `disjoint_tree_overlay` reflecting these action constructors. ## Action graph data structure: actions of overlaying trees The main components of the action graph are - `"actions"`, describing how new artifacts can be obtained by running a command in a directory given by arranging existing artifacts in a specified way, - `"blobs"`, strings that can later be referenced as "known" artifacts through their content-addressable blob identifier, and - `"trees"`, directory objects given by an arrangement of already existing artifacts. Additionally, there is the category of `"tree overlays"` mapping (intensional) names to their definition as a list of existing tree artifacts, together with a bit indicating if the build should be aborted if the overlaid trees conflict on any path. The extensional value of such a tree overlay is obtained by starting with the empty tree and, sequentially in the given order, overlay the extensional value of the defining artifacts. Here, the overlay of one tree by another is a tree where the maximal paths are those of the second tree together with those of the first tree that are not in conflict with any from the second; the artifact at such a maximal path is the one at that place in the second tree if the second tree contains this maximal path, otherwise the artifact at this position in the first tree. ## Computation of `"tree overlays"` in the presence of remote execution The evaluation of `"tree overlays"` happens in memory in the `just` process. To do so, the actual tree objects have to be inspected, in fact downwards for all common paths. In particular, as opposed to all other operations, trees in this operation cannot be passed on as opaque objects by simply copying the identifier. In the case of remote execution that means that the respective tree objects have to be fetched; to avoid unnecessary traffic, only the needed tree objects are fetched without the blobs or tree objects outside common paths, even if that means that those objects cannot be put into the local CAS (as that would violate the tree invariant). In any case, when adding the new tree objects that are part of the overlaid tree, `just` adds them to the applicable CAS in topological order, in order to keep the tree invariant. ## Functions in rule definition: `TREE_OVERLAY` and `DISJOINT_TREE_OVERLAY` In the defining expressions of rules, the constructors `TREE_OVERLAY` and `DISJOINT_TREE_OVERLAY` can be used to describe parts of the action graph. These constructors have one argument `"$1"` which has to evaluate to a list of tree-conflict—free mappings of strings to artifacts, also called "stages". The result of this function is a single artifact, the tree defined to be the overlay or conflict-free overlay, respectively, of the trees corresponding to the stages. The reason we require stages to be passed to these constructors rather than artifacts that happen to be trees is twofold. - We want to find malformed expressions already analysis time; therefore, we need to ensure not only that the arguments to the `"tree_overlays"` entry in the action graph are artifacts, but, in fact, tree artifacts. By requiring that implicit tree constructor we avoid accidental use of file outputs, as a location has to be explicitly specified. - One the other hand, we expect that often the inputs are the artifacts of a dependency, which is naturally given as a stage via `DEP_ARTIFACTS`. So this form of definition is actually more convenient to use. ## Built-in functions `tree_overlay` and `disjoint_tree_overlay` As for any build primitive there is also a corresponding built-in rule type. The rules `"tree_overlay"` and `"disjoint_tree_overlay"` have a single field `"deps"` which expects a list of targets. Both runfiles and artifacts of the `"tree_overlay"` target are the tree overlays or conflict-free tree overlays, respectively, of the artifacts of the specified `"deps"` targets in the specified order staged at the value of the field `"name"` which has to evaluate to a single string. just-buildsystem-justbuild-b1fb5fa/doc/future-designs/000077500000000000000000000000001516554100600233305ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/future-designs/README.md000066400000000000000000000006061516554100600246110ustar00rootroot00000000000000This directory contains a collection of design documents for planned additions and changes to the `just` build tool. In this way, the design of upcoming larger changes can be reviewed in a transparent way, before the implementation is started. The design documents in this directory are not yet implemented, and there is no promise of when those changes will be implemented, or if at all. just-buildsystem-justbuild-b1fb5fa/doc/future-designs/cas-objects-import.md000066400000000000000000000106751516554100600273700ustar00rootroot00000000000000Importing objects to CAS ======================== Motivation ---------- Roots in `just` builds are typically Git trees. Such artifacts should ideally be available already in CAS to avoid any expensive fetches. It is not uncommon for checkouts or other pre-fetched data to be present locally and to want to quickly integrate them into a multi-repository build. A typical approach is to pack the process that generates the root directory and all its content into a script (or a sequence of commands) and place it as the fetch command of a `git tree` repository inside the multi-repository configuration file provided to `just-mr`. This however comes with two inefficiencies: firstly, the user has to specify a priori the expected Git tree identifier of the root, and secondly, if the tree is not already cached then `just-mr` will have to (re)run the (usually expensive) fetch command and hash the generated directory in order to store the root in its Git cache. Implemented Proposal -------------------- We propose a new `just add-to-cas` subcommand which takes in a filesystem path and provides its Git hash to standard output. A corresponding entry is also added to the local (file, executable, or tree) CAS. While hashing directories into Git trees is the most important use-case, the subcommand will allow the hashing of blobs as well. Note that `just-mr` by default looks both into the local CAS and the Git cache for any blobs or trees it might need to make available. In particular, build roots with trees already in local CAS would only need to be additionally imported into the Git cache, which is a cheap local operation. ### CAS locations `just` provides a default local build root location for where the local CAS should be stored. However, a useful use-case of this new subcommand would be to populate a specific (and possibly existing) CAS. Therefore an option to specify the local build root will be available. ### Remote CAS Similarly, we want to be able to populate a remote CAS as well, e.g., to prepare roots for a future build. For this reason, the subcommand will allow to specify a remote-execution endpoint, with the understanding that the generated local CAS entry should be synced with the remote. Additionally, we will support computing hashes also in compatible mode, set via an appropriate option. ### Symbolic links If the given path points to a symbolic link, by default the link content will be added to the CAS as blob. If the the option `--follow-symlinks` is given, the argument specifying what to add will be resolved and the object pointed to will be added to CAS. If the given path points to a directory, non-upwards symbolic links will be accepted and added to the tree object. Adding directories to CAS is only suppored in native mode. ### Other notes The only mandatory argument is the path to the filesystem object to be hashed, with all other options optional, as sensible defaults exist already in `just`. Logging options will also be available for this subcommand. While the main purpose of this subcommand is to add the hashed Git object to CAS, and considering that hashing a directory is a non-trivial operation for the command-line `git` tool, a pure computation of the hashes _without_ generating a CAS entry might still be of interest and available as an option. This is, however, not useful in typical situations. ### Auxillary change: `just install-cas --archive` Build sources many times come in the form of archives, which allow efficient storage and backup, and permit easy versioning of build dependencies. Also build artifacts, for example resulting binaries (together with required headers when building statically), are typically shipped as archives. Therefore, the `just install-cas` command has an option to dump an artifact, that has to be a tree, as an archive instead of as a directory (or pretty-printed top-level contents). Remote CAS options and file location (with stdout as default) are honored. Auxiliary changes still to be implemented ----------------------------------------- ### `just add-to-cas` to support symlink resolving inside directories To mirror options available in `just-mr` repository descriptions, we will also support options to either ignore, or fully resolve symlinks in the generated Git trees. ### `just install-cas --archive` to support different archive types By default, for reproducibility reasons, the archiving format will be a tarball. Options will be added to produce other archive types, as supported by `just-mr`. just-buildsystem-justbuild-b1fb5fa/doc/future-designs/upwards-symlinks.md000066400000000000000000000141721516554100600272130ustar00rootroot00000000000000# Support for upwards symbolic links ## Background ### Existing symlink support Our build system already supports non-upwards relative symbolic links as first-class objects. The reason for choosing this restriction was that those can be placed anywhere inside an action directory without that directory becoming dependent on the ambient system, regardless if symbolic links are followed or inspected via readlink(2). Additionally, `just-mr` supports resolving symlinks that are entirely within a logical root. This allows to support roots where symlinks are used to have file under version control only once; as roots are internally represented in a content-addressable way, the duplication implicit to the resolving comes at no extra cost. ### Level of enforcement The restriction to only allow only non-upwards relative symlinks is enforced - in the output of actions, including output directories, - in explicit tree references on `file` roots. However, it is not enforced when using an explicit tree reference on a `git` root as input. The reason is that we want to benefit from the fact that trees are already hashed in git repositories so that we can get values (and harvest cache hits) without ever looking at that subdirectory. ### Dangling relative upwards symbolic links While used rarely, dangling upwards relative symlinks do exist in some projects, also for legitimate reasons. Being dangling, they cannot be resolved in the root definition. Where such projects occur as (transitive) dependency, some form of extended symlink handling seems desirable. ## Proposed changes ### Extend definition language to support arbitrary relative symlinks We extend the description language to allow handling relative symlinks, including upward ones, in a safe way. #### Extensional symlink level of a computed artifact We associate a symlink-level with blobs and trees that do not transitively contain absolute symlinks in the following way. - Files and executable files have symlink level 0. - The symlink level of a relative symlink is the number of (necessarily leading) `../` of the syntactical canonical form, when read as a relative path. - The symlink level of a tree is the maximum of 0 and the symlink-levels of all (immediate) subobjects reduced by one. #### The declared symlink level In the description language a declared symlink level is associated with each artifact; the invariant is that - the declared level is always at least as big as the extensional level of the defined object, and - the action directory of each action has declared level 0, ensuring that action directories do not refer to external sources. In order to do so, we extend our target names for explicit symlink and tree references to optionally (defaulting to 0) declare a symlink level. This is done by allowing in the target name, an additional dict as last entry, which may contain the key `"symlink level"`; the value for that key is the declared level and has to be a non-negative integer. For example, - `["SYMLINK", null, "foo", {"symlink level": 3}]` is an explicit symlink reference to a symlink at `foo` in the current module with declared symlink level 3, and - `["TREE", null, "bar", {"symlink level": 4}]` is an explicit tree reference to a tree located at `bar` in the current module with declared symlink level 4. The reason for choosing a dict rather than a positional argument is to be prepared for additional declared properties in the future, should they become necessary. We extend the definition of `"ACTION"` function available inside rule definitions to declare a symlink level of output files and output directories by allowing an optional map `"symlink level"` with keys the names of the declared outputs (files and directories) and values non-negative integers. The declared symlink level of an action artifact is the value of the output path in that map; if not found in that map, the declared value is 0; in this way, this extension is backwards compatible. Moreover, we require that in the input stage of an action, every artifact of declared symlink level `n` is staged under at least `n` directories. We also extend the `"SYMLINK"` function available inside rule definitions to have an optional field `"symlink level"` specifying the declared symlink level. The value, if specified, has to be a non-negative integer and the default is 0. Artifacts defined by the `"TREE"` function have as declared symlink level the maximum of 0 and for each artifact of the defining stage the difference of the symlink level of the artifact and the number of directories it is staged under. Artifacts defined by the `"BLOB"` function available inside rule definitions have symlink level 0. ### Enforce correctness of symlink level All computed artifacts (that are `KNOWN` artifacts once computed) carry the actual symlink level as part of their data structure; when serializing the artifact description, the symlink level is reported if (and only if) it is not 0. For inputs (explicit symlink and tree reference) we enforce that the actual symlink level is not larger than the declared one. To enforce this check while keeping the benefits of explicit tree references in `git` roots, we keep a cache (in the form of a simple mapping on disk) of tree identifiers to their symlink level. This map will be garbage collected together with the repository roots (as described in the "Garbage collection for Repository Roots" design). We also enforce the correctness of the symlink level of action outputs: for `"outs"`, if they are a symlink, the level is checked and the action rejected if the actual level is larger than the declared one. For `"out_dirs"`, the directory is rejected if the actual symlink level is larger than the declared one. It then follows that all action have actual symlink level 0 of the input stage. ### Extensional projection reduces symlink level In evaluation of an export target, the intensional description with the declared symlink level is replaced by the extensional description using the extensional symlink level. By our construction, the latter is less or equal than the former. Hence, this projections allows at most _more_ builds which is in line with the properties of export targets. just-buildsystem-justbuild-b1fb5fa/doc/images/000077500000000000000000000000001516554100600216315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/images/justbuild-logo.svg000066400000000000000000000032721516554100600253210ustar00rootroot00000000000000 编组 120备份 12 just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/000077500000000000000000000000001516554100600252015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/server.py000077500000000000000000000743241516554100600270760ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import jinja2 import json import re import os import subprocess import werkzeug.exceptions import werkzeug.routing import werkzeug.utils from werkzeug.wrappers import Request, Response from werkzeug.middleware.shared_data import SharedDataMiddleware from functools import cache def core_config(conf): new_conf = {} for k, v in conf.items(): if v is not None: new_conf[k] = v return new_conf class HexIdentifierConverter(werkzeug.routing.BaseConverter): regex = '[a-zA-Z0-9]{2,300}' class HashIdentifierConverter(werkzeug.routing.BaseConverter): regex = '[a-zA-Z0-9]{40,64}' class InvocationIdentifierConverter(werkzeug.routing.BaseConverter): regex = '[-:_a-zA-Z0-9]{1,200}' class FileIdentifierConverter(werkzeug.routing.BaseConverter): regex = '[-:_.a-zA-Z0-9]{3,200}' class InvocationServer: MAX_FULL_INVOCATIONS = 50 MAX_REDUCED_INVOCATIONS = 450 def __init__(self, logsdir, *, just_mr = None, graph = "graph.json", artifacts = "artifacts.json", artifacts_to_build = "to-build.json", profile = "profile.json", meta = "meta.json"): self.logsdir = logsdir if just_mr is None: self.just_mr = ["just-mr"] else: self.just_mr = just_mr self.profile = profile self.graph = graph self.artifacts = artifacts self.artifacts_to_build = artifacts_to_build self.meta = meta self.templatepath = os.path.join(os.path.dirname(__file__), "templates") self.jinjaenv = jinja2.Environment( loader=jinja2.FileSystemLoader(self.templatepath)) rule = werkzeug.routing.Rule self.routingmap = werkzeug.routing.Map([ rule("/", methods=("GET",), endpoint="list_invocations"), rule("/filterinvocations/context//", methods=("GET",), endpoint="filter_context"), rule("/filterinvocations/noncached", methods=("GET",), endpoint="filter_noncached"), rule("/filterinvocations/remote/prop//", methods=("GET",), endpoint="filter_remote_prop"), rule("/filterinvocations/remote/address/", methods=("GET",), endpoint="filter_remote_address"), rule("/filterinvocations/target/", methods=("GET",), endpoint="filter_target"), rule("/blob/", methods=("GET",), endpoint="get_blob"), rule("/blob//", methods=("GET",), endpoint="get_blob_as"), rule("/tree/", methods=("GET",), endpoint="get_tree"), rule("/invocations/", methods=("GET",), endpoint="get_invocation"), rule("/critical_path/", methods=("GET",), endpoint="get_critical_path"), ], converters=dict( invocationidentifier=InvocationIdentifierConverter, hashidentifier=HashIdentifierConverter, hexidentifier=HexIdentifierConverter, fileidentifier=FileIdentifierConverter, )) @Request.application def __call__(self, request): mapadapter = self.routingmap.bind_to_environ(request.environ) try: endpoint, args = mapadapter.match() return getattr(self, "do_%s" % (endpoint,))(**args) except werkzeug.exceptions.HTTPException as e: return e def render(self, templatename, params): response = Response() response.content_type = "text/html; charset=utf8" template = self.jinjaenv.get_template(templatename) response.data = template.render(params).encode("utf8") return response def do_list_invocations(self, *, filter_info="", profile_filter = lambda p : True, metadata_filter = lambda p : True): entries = sorted(os.listdir(self.logsdir), reverse=True) context_filters = {} remote_props_filters = {} remote_address_filters = set() target_filters = set() def add_filter(data, filters): if isinstance(filters, set): filters.add(json.dumps(data)) elif isinstance(filters, dict): for k, v in data.items(): key = json.dumps(k) value = json.dumps(v) if key not in filters: filters[key] = set() filters[key].add(value) # Load full invocation data (profile data + metadata). index = 0 full_invocations = [] full_invocations_count = 0 while index < len(entries): e = entries[index] index += 1 profile = os.path.join(self.logsdir, e, self.profile) if not os.path.exists(profile): # only consider invocations that provide a profile; unfinished # invocations as well as not build related ones (like # install-cas) are not relevant. continue try: with open(profile) as f: profile_data = json.load(f) except: profile_data = {} meta = os.path.join(self.logsdir, e, self.meta) try: with open(meta) as f: meta_data = json.load(f) except: meta_data = {} if (not profile_filter(profile_data)) or (not metadata_filter(meta_data)): continue full_invocations_count += 1 target = profile_data.get("target") build_start_time = profile_data.get("build start time") build_stop_time = profile_data.get("build stop time") stop_time = profile_data.get("stop time") build_wall_clock_time = build_stop_time - build_start_time if (build_start_time is not None and build_stop_time is not None) else None config = core_config(profile_data.get("configuration", {})) context = meta_data.get("context", {}) remote = profile_data.get('remote', {}) remote_props = remote.get('properties', {}) remote_address = remote.get('address') add_filter(context, context_filters) add_filter(remote_props, remote_props_filters) add_filter(remote_address, remote_address_filters) add_filter(target, target_filters) invocation = { "name": e, "subcommand": profile_data.get("subcommand"), "target": json.dumps(target) if target else None, "config": json.dumps(config) if config else None, "context": json.dumps(context) if context else None, "build_wall_clock_time": "%5ds" % (build_wall_clock_time,) if build_wall_clock_time else None, "exit_code": profile_data.get('exit code', 0), "remote_address": remote_address, "remote_props": json.dumps(remote_props) if remote_props else None, } full_invocations.append(invocation) if full_invocations_count >= InvocationServer.MAX_FULL_INVOCATIONS: break # Load reduced invocation data (metadata only). reduced_invocations = [] reduced_invocations_count = 0 if not filter_info: while index < len(entries): e = entries[index] index += 1 profile = os.path.join(self.logsdir, e, self.profile) if not os.path.exists(profile): # only consider invocations that provide a profile; unfinished # invocations as well as not build related ones (like # install-cas) are not relevant. continue meta = os.path.join(self.logsdir, e, self.meta) try: with open(meta) as f: meta_data = json.load(f) except: meta_data = {} reduced_invocations_count += 1 context = meta_data.get("context", {}) add_filter(context, context_filters) invocation = { "name": e, "context": json.dumps(context) if context else None, } reduced_invocations.append(invocation) if reduced_invocations_count >= InvocationServer.MAX_REDUCED_INVOCATIONS: break def convert_filters(filters): def convert_values(values): return sorted([{ "value": v, "value_hex": v.encode().hex(), } for v in values], key=lambda x: x["value"]) if isinstance(filters, set): return convert_values(filters) if len(filters) > 1 else [] elif isinstance(filters, dict): return sorted([{ "key": key, "key_hex": key.encode().hex(), "values": convert_values(values) } for key, values in filters.items() if len(values) > 1], key=lambda x: x["key"]) else: return [] return self.render("invocations.html", {"full_invocations": full_invocations, "full_invocations_count": full_invocations_count, "reduced_invocations": reduced_invocations, "reduced_invocations_count": reduced_invocations_count, "filter_info": filter_info, "context_filters": convert_filters(context_filters), "remote_props_filters": convert_filters(remote_props_filters), "remote_address_filters": convert_filters(remote_address_filters), "target_filters": convert_filters(target_filters)}) def do_filter_remote_prop(self, key, value): filter_info = "remote-execution property" try: key_string = json.loads(bytes.fromhex(key).decode('utf-8')) value_string = json.loads(bytes.fromhex(value).decode('utf-8')) filter_info += " " + json.dumps({key_string: value_string}) except: pass def check_prop(p): for k, v in p.get('remote', {}).get('properties', {}).items(): if (json.dumps(k).encode().hex() == key) and (json.dumps(v).encode().hex() == value): return True return False return self.do_list_invocations( filter_info = filter_info, profile_filter = check_prop, ) def do_filter_remote_address(self, value): filter_info = "remote address" try: value_string = json.loads(bytes.fromhex(value).decode('utf-8')) filter_info += " " + json.dumps(value_string) except: pass def check_prop(p): v = p.get('remote', {}).get('address') return (json.dumps(v).encode().hex() == value) return self.do_list_invocations( filter_info = filter_info, profile_filter = check_prop, ) def do_filter_context(self, key, value): filter_info = "context variable" try: key_string = json.loads(bytes.fromhex(key).decode('utf-8')) value_string = json.loads(bytes.fromhex(value).decode('utf-8')) filter_info += " " + json.dumps({key_string: value_string}) except: pass def check_prop(p): for k, v in p.get('context', {}).items(): if (json.dumps(k).encode().hex() == key) and (json.dumps(v).encode().hex() == value): return True return False return self.do_list_invocations( filter_info = filter_info, metadata_filter = check_prop, ) def do_filter_target(self, value): filter_info = "target" try: value_string = json.loads(bytes.fromhex(value).decode('utf-8')) filter_info += " " + json.dumps(value_string) except: pass def check_prop(p): v = p.get('target') return (json.dumps(v).encode().hex() == value) return self.do_list_invocations( filter_info = filter_info, profile_filter = check_prop, ) def do_filter_noncached(self): def check_noncached(p): for v in p.get('actions', {}).values(): if not v.get('cached'): return True return False return self.do_list_invocations( filter_info = "not fully cached", profile_filter = check_noncached, ) def process_failure(self, cmd, procobj, *, failure_kind=None): params = {"stdout": None, "stderr": None, "failure_kind": failure_kind} params["cmd"] = json.dumps(cmd) params["exit_code"] = procobj.returncode try: params["stdout"] = procobj.stdout.decode('utf-8') except: pass try: params["stderr"] = procobj.stderr.decode('utf-8') except: pass return self.render("failure.html", params) def do_get_blob(self, blob, *, download_as=None): cmd = self.just_mr + ["install-cas", "--remember", blob] blob_data = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if blob_data.returncode !=0: return self.process_failure(cmd, blob_data) try: blob_content = blob_data.stdout.decode('utf-8') except: # Not utf-8, so return as binary file to download separately download_as = download_as or blob if download_as: response = Response() response.content_type = "application/octet-stream" response.data = blob_data.stdout response.headers['Content-Disposition'] = \ "attachement; filename=%s" %(download_as,) return response return self.render("blob.html", {"name": blob, "data": blob_content}) def do_get_blob_as(self, blob, name): return self.do_get_blob(blob, download_as=name) def do_get_tree(self, tree): cmd = self.just_mr + ["install-cas", "%s::t" % (tree,)] tree_data = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if tree_data.returncode !=0: return self.process_failure(cmd, tree_data) try: tree_info = json.loads(tree_data.stdout.decode('utf-8')) except Exception as e: return self.process_failure( cmd, tree_data, failure_kind = "Malformed output: %s" % (e,)) entries = [] for k, v in tree_info.items(): h, s, t = v.strip('[]').split(':') entries.append({"path": k, "hash": h, "type": "tree" if t == 't' else "blob"}) return self.render("tree.html", {"name": tree, "entries": entries}) def do_get_invocation(self, invocation): params = {"invocation": invocation, "have_profile": False} try: with open(os.path.join( self.logsdir, invocation, self.profile)) as f: profile = json.load(f) params["have_profile"] = True except: profile = {} try: with open(os.path.join( self.logsdir, invocation, self.meta)) as f: meta = json.load(f) except: meta = {} try: with open(os.path.join( self.logsdir, invocation, self.graph)) as f: graph = json.load(f) except: graph = {} try: with open(os.path.join( self.logsdir, invocation, self.artifacts)) as f: artifacts = json.load(f) except: artifacts = {} params["repo_config"] = meta.get('configuration') params["exit_code"] = profile.get('exit code') analysis_errors = [] blob_pattern = re.compile(r'blob ([0-9A-Za-z]{40,64})') for s in profile.get('analysis errors', []): analysis_errors.append({ "msg": s, "blobs": blob_pattern.findall(s), }) params["analysis_errors"] = analysis_errors params["has_remote"] = profile.get('remote') is not None remote_address = profile.get('remote', {}).get('address') params["is_remote"] = remote_address is not None params["remote_address"] = { "value": json.dumps(remote_address), "value_hex": json.dumps(remote_address).encode().hex(), } remote_props = [] for k, v in profile.get('remote', {}).get('properties', {}).items(): key = json.dumps(k) value = json.dumps(v) remote_props.append({ "key": key, "key_hex": key.encode().hex(), "value": value, "value_hex": value.encode().hex(), }) params["remote_props"] = remote_props remote_dispatch = profile.get('remote', {}).get('dispatch', []) params["remote_dispatch"] = json.dumps(remote_dispatch) if remote_dispatch else None # For complex conditional data fill with None as default for k in ["cmdline", "cmd", "target", "config"]: params[k] = None # Fill this data, if available if meta.get('cmdline'): params["cmdline"] = json.dumps(meta.get('cmdline')) context = [] for k, v in meta.get('context', {}).items(): key = json.dumps(k) value = json.dumps(v) context.append({ "key": key, "key_hex": key.encode().hex(), "value": value, "value_hex": value.encode().hex(), }) params["context"] = context if profile.get('subcommand'): params["cmd"] = json.dumps( [profile.get('subcommand')] + profile.get('subcommand args', [])) target = profile.get('target') if target: params["target"] = { "value": json.dumps(target), "value_hex": json.dumps(target).encode().hex() } if profile.get('configuration') is not None: params["config"] = json.dumps(core_config( profile.get('configuration'))) output_artifacts = [] for k, v in artifacts.items(): output_artifacts.append({ "path": k, "basename": os.path.basename(k), "hash": v["id"], "type": "tree" if v["file_type"] == "t" else "blob", }) params["artifacts"] = output_artifacts params["artifacts_count"] = len(output_artifacts) actions_considered = set() failed_build_actions = [] failed_test_actions = [] for k, v in profile.get('actions', {}).items(): if v.get('exit code', 0) != 0: actions_considered.add(k) if graph.get('actions', {}).get(k, {}).get('may_fail') != None: failed_test_actions.append(action_data(k, v, graph)) else: failed_build_actions.append(action_data(k, v, graph)) params["failed_actions"] = failed_build_actions + failed_test_actions params["failed_actions_count"] = len(params["failed_actions"]) # non-failed actions with output output_actions = [] for k, v in profile.get('actions', {}).items(): if k not in actions_considered: if v.get('stdout') or v.get('stderr'): actions_considered.add(k) output_actions.append(action_data(k, v, graph)) params["output_actions"] = output_actions params["output_actions_count"] = len(output_actions) # longest running non-cached non-failed actions without output candidates = [] action_count = 0 action_count_cached = 0 for k, v in profile.get('actions', {}).items(): action_count += 1 if not v.get('cached'): if k not in actions_considered: candidates.append((v.get('duration', 0.0), k, v)) else: action_count_cached += 1 params["action_count"] = action_count params["action_count_cached"] = action_count_cached params["fully_cached"] = (action_count == action_count_cached) candidates.sort(reverse=True) non_cached = [] params["more_noncached"] = None params["non_cached_count"] = len(candidates) if len(candidates) > 30: params["more_non_cached"] = len(candidates) - 30 candidates = candidates[:30] for t, k, v in candidates: action = action_data(k, v, graph) action["name_prefix"] = "%5.1fs" % (t,) non_cached.append(action) params["non_cached"] = non_cached start_time = profile.get("start time") build_start_time = profile.get("build start time") build_stop_time = profile.get("build stop time") stop_time = profile.get("stop time") wall_clock_time = stop_time - start_time if (start_time is not None and stop_time is not None) else None build_wall_clock_time = build_stop_time - build_start_time if (build_start_time is not None and build_stop_time is not None) else None params["wall_clock_time"] = "%ds" % (wall_clock_time,) if wall_clock_time else None params["build_wall_clock_time"] = "%ds" % (build_wall_clock_time,) if build_wall_clock_time else None return self.render("invocation.html", params) def do_get_critical_path(self, invocation): params = {"invocation": invocation} try: with open(os.path.join( self.logsdir, invocation, self.profile)) as f: profile = json.load(f) except: profile = {} try: with open(os.path.join( self.logsdir, invocation, self.graph)) as f: graph = json.load(f) except: graph = {} try: with open(os.path.join( self.logsdir, invocation, self.artifacts_to_build)) as f: artifacts_to_build = json.load(f) except: artifacts_to_build = {} @cache def critical_path(artifact_type, source_id): input_artifacts = [] source_duration = 0.0 if artifact_type == "ACTION": input_artifacts = graph.get("actions", {}).get(source_id, {}).get("input", {}).values() source_duration = profile.get("actions", {}).get(source_id, {}).get("duration", 0.0) elif artifact_type == "TREE": input_artifacts = graph.get("trees", {}).get(source_id, {}).values() elif artifact_type == "TREE_OVERLAY": input_artifacts = graph.get("tree_overlays", {}).get(source_id, {}).get("trees", []) else: return ([], 0.0) max_path = [] max_duration = 0.0 for artifact_desc in input_artifacts: path, duration = critical_path( artifact_desc.get("type", {}), artifact_desc.get("data", {}).get("id")) if duration > max_duration: max_duration = duration max_path = path path = max_path + [(artifact_type, source_id)] duration = max_duration + source_duration return (path, duration) max_path = [] max_name = None max_duration = 0.0 for artifact_name, artifact_desc in artifacts_to_build.items(): path, duration = critical_path( artifact_desc.get("type", {}), artifact_desc.get("data", {}).get("id")) if duration > max_duration: max_duration = duration max_path = path max_name = artifact_name max_path_data = [] for artifact_type, source_id in reversed(max_path): source_data = {} if artifact_type == "ACTION": action_profile = profile.get('actions', {}).get(source_id, {}) source_data = action_data(source_id, action_profile, graph) source_data["name_prefix"] = "%5.1fs" % (action_profile.get("duration", 0.0),) else: source_data = {"name": source_id} max_path_data.append({ "type": artifact_type, "data": source_data }) build_start_time = profile.get("build start time") build_stop_time = profile.get("build stop time") build_wall_clock_time = build_stop_time - build_start_time if (build_start_time is not None and build_stop_time is not None) else None params["critical_artifact"] = { "name": max_name, "path": max_path_data if max_path_data else None, "duration": '%0.3fs' % (max_duration,) if max_duration > 0 else None, "build_wall_clock_time": "%ds" % (build_wall_clock_time,) if build_wall_clock_time else None, } return self.render("critical_path.html", params) def action_data(name, profile_value, graph): data = { "name_prefix": "", "name": name, "cached": profile_value.get('cached'), "exit_code": profile_value.get('exit code', 0)} duration = profile_value.get('duration', 0.0) data["duration"] = '%0.3fs' % (duration,) if duration > 0 else None desc = graph.get('actions', {}).get(name, {}) cmd = desc.get('command', []) data["cmd"] = json.dumps(cmd) if cmd else None data["may_fail"] = desc.get("may_fail") data["stdout"] = profile_value.get('stdout') data["stderr"] = profile_value.get('stderr') data["output"] = desc.get('output', []) data["output_dirs"] = desc.get('output_dirs', []) if len(data["output"]) == 1: data["primary_output"] = data["output"][0] elif len(data["output"]) == 0 and len(data["output_dirs"]) == 1: data["primary_output"] = data["output_dirs"][0] else: data["primary_output"] = None data["artifacts"] = profile_value.get('artifacts', {}) data["basenames"] = { name : os.path.basename(name) for name in data["artifacts"].keys() } origins = [] for origin in desc.get('origins', []): origins.append("%s#%d@%s" % ( json.dumps(origin.get('target', [])), origin.get('subtask', 0), core_config(origin.get('config', {})),)) data["origins"] = origins return data def create_app(logsdir, **kwargs): app = InvocationServer(logsdir, **kwargs) app = SharedDataMiddleware(app, { '/static': os.path.join(os.path.dirname(__file__), 'static') }) return app if __name__ == '__main__': import sys from argparse import ArgumentParser from wsgiref.simple_server import make_server parser = ArgumentParser( description="Serve invocations of a given directory") parser.add_argument("--meta", dest="meta", default="meta.json", help="Name of the logged metadata file") parser.add_argument("--graph", dest="graph", default="graph.json", help="Name of the logged action-graph file") parser.add_argument("--artifacts", dest="artifacts", default="artifacts.json", help="Name of the logged artifacts file") parser.add_argument("--artifacts-to-build", dest="artifacts_to_build", default="to-build.json", help="Name of the logged artifacts-to-build file") parser.add_argument("--profile", dest="profile", default="profile.json", help="Name of the logged profile file") parser.add_argument( "--just-mr", dest="just_mr", default='["just-mr"]', help="The correct way of invoking just-mr, as JSON vector of strings") parser.add_argument("-i", dest="interface", default='127.0.0.1', help="The interface to listen on") parser.add_argument("-p", dest="port", default=9000, type=int, help="The port to listen on") parser.add_argument("DIR", help="Directory (for a single project id) to serve") args = parser.parse_args() try: just_mr = json.loads(args.just_mr) except Exception as e: print("just-mr argument should be valid json, but %r is not: %s" % (args.just_mr, e), file=sys.stderr) sys.exit(1) app = create_app(args.DIR, just_mr=just_mr, graph=args.graph, artifacts=args.artifacts, artifacts_to_build=args.artifacts_to_build, meta=args.meta, profile=args.profile) make_server(args.interface, args.port, app).serve_forever() just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/static/000077500000000000000000000000001516554100600264705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/static/style.css000066400000000000000000000006421516554100600303440ustar00rootroot00000000000000body { background: #e8eff0; font-family: "Helvetica"; font-size: large; } h1, h2, h3 { color: #11557c; } #invocation-list > ul > li { box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05); border: 1px solid #b2bec3; border-radius: 6px; margin-bottom: 15px; padding: 3px 7px; } #invocation-verb { font-weight: 700; } #invocation-target { background-color: #ced6d9; } #invocation-failure { color: #d63031; } just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/000077500000000000000000000000001516554100600271775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/base.html000066400000000000000000000005031516554100600307750ustar00rootroot00000000000000 {% block heading %}Invocation Browser{% endblock %} {% block content %} (no content) {% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/blob.html000066400000000000000000000003471516554100600310070ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Blob {{ name | e }} {% endblock %} {% block content %}

Blob {{ name | e }}

download 
{{ data | e}}
{% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/critical_path.html000066400000000000000000000021531516554100600326740ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Invocations {{invocation | e}} {% endblock %} {% from 'macros.html' import show_action %} {% block content %}

Critical path for {{ invocation | e }}

Overview

  • Invocation: {{ invocation | e }}
    • {% if critical_artifact["build_wall_clock_time"] %}
  • Wall-clock time (build only): {{ critical_artifact["build_wall_clock_time"] | e }}
    • {% endif %} {% if critical_artifact["duration"] %}
  • Length of critical path: {{ critical_artifact["duration"] | e }}
    • {% endif %} {% if critical_artifact["name"] %}
  • Critical artifact: {{ critical_artifact["name"] | e }}
    • {% endif %} {% if critical_artifact["path"] %}
  • Path:
      • {% for entry in critical_artifact["path"] %} {% if entry["type"] == "ACTION" %} {{ show_action(entry["data"]) }} {% else %}
    • {{ entry["type"] | lower | e }} {{ entry["data"]["name"] | e }}
      • {% endif %} {% endfor %}
    {% endif %}
{% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/failure.html000066400000000000000000000010221516554100600315070ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Failure calling an external process {% endblock %} {% block content %}

Failure calling an external process

    {% if failure_kind %}
  • {{ failure_kind | e }}
    • {% endif %}
  • Command: {{ cmd|e }}
  • exit code: {{ exit_code|e }}
    • {% if stdout %}
  • stdout: 
    {{ stdout | e }}
    • {% endif %} {% if stderr %}
  • stderr: 
    {{ stderr | e }}
    • {% endif %}
{% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/invocation.html000066400000000000000000000111661516554100600322430ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Invocation {{invocation | e}} {% endblock %} {% from 'macros.html' import show_action %} {% block content %}

Invocation {{invocation | e}}

(all invocations)

Overview

    {% if cmd %}
  • Subcommand and positional arguments: {{ cmd | e }}
    • {% endif %} {% if cmdline %}
  • Full command line {{ cmdline | e }}
    • {% endif %} {% if repo_config %}
  • Repository configuration: {{ repo_config | e}}
    • {% endif %} {% if target %}
  • Target: {{ target["value"] | e }}
    • {% endif %} {% if config %}
  • Target configuration: {{ config | e }}
    • {% endif %} {% if has_remote %}
  • {% if is_remote %} Remote build: {% else %} Local build: {% endif %}
    • {% endif %} {% if context %}
  • Context:
    • {% else %}
  • Context: (none)
    • {% endif %} {% if wall_clock_time %}
  • Wall-clock time: {{ wall_clock_time | e }}
    • {% endif %} {% if build_wall_clock_time %}
  • Wall-clock time (build only): {{ build_wall_clock_time | e }}
    • {% endif %} {% if exit_code != None %}
  • Exit code: {{ exit_code | e }}
    • {% endif %} {% if not fully_cached %}
  • Critical path
    • {% endif %} {% if artifacts %}
  • {% if artifacts_count == 1 %}
    artifact {% else %}
    {{ artifacts_count | e }} artifacts {% endif %}
    • {% endif %}
{% if have_profile %} {% if analysis_errors %}

Analysis errors

    {% for error in analysis_errors %} 
  • {{ error["msg"] | e}}
    {% if error["blobs"] %} Blobs: {% for blob in error["blobs"] %} {{ blob | e }} {% endfor %} {% endif %}
    • {% endfor %}
{% else %} {% if exit_code == 0 or exit_code == 1 or exit_code == 2 %}

Actions

  • Processed: {{ action_count | e }}
  • Cached: {{ action_count_cached | e }}

Failed actions ({{ failed_actions_count | e }})

{% if failed_actions %}
    {% for action in failed_actions %} {{ show_action(action, loop.index == 1) }} {% endfor %}
{% else %} (none) {% endif %}

Other actions with console output ({{ output_actions_count | e }})

{% if output_actions %}
    {% for action in output_actions %} {{ show_action(action) }} {% endfor %}
{% else %} (none) {% endif %}

Remaining non-cached actions ({{ non_cached_count | e}})

{% if non_cached %} In order of decreasing run time.
    {% for action in non_cached %} {{ show_action(action) }} {% endfor %} {% if more_non_cached %}
  • … and {{ more_non_cached | e }} actions
    • {% endif %}
{% else %} (none) {% endif %} {% endif %} {% endif %} {% else %} No profiling data available; invocation data is not (yet) complete. {% endif %} {% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/invocations.html000066400000000000000000000077321516554100600324320ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Recent Invocations {% endblock %} {% block content %} {% if filter_info %}

Invocations filtered by {{ filter_info | e }}

(all invocations) {% else %}

Recent Invocations

Filter by: only not fully cached {% if context_filters %}
Context {% for entry in context_filters %} {{ entry.key | e }}: {% for value in entry["values"] %} {% if loop.index > 1 %} , {% endif %} {{ value["value"] | e }} {% endfor %}
{% endfor %}
{% endif %} {% if remote_props_filters %}
Remote properties {% for entry in remote_props_filters %} {{ entry.key | e }}: {% for value in entry["values"] %} {% if loop.index > 1 %} , {% endif %} {{ value["value"] | e }} {% endfor %}
{% endfor %}
{% endif %} {% if remote_address_filters %}
Remote address {% for entry in remote_address_filters %} {% if loop.index > 1 %} , {% endif %} {{ entry["value"] | e }} {% endfor %}
{% endif %} {% if target_filters %}
Target {% for entry in target_filters %} {% if loop.index > 1 %} , {% endif %} {{ entry["value"] | e }} {% endfor %}
{% endif %} {% endif %} {% if full_invocations %}
    {% for invocation in full_invocations %}
  • {{ invocation.name | e }} {% if invocation.subcommand %} {{ invocation.subcommand | e }} {% endif %} {% if invocation.target %} {{ invocation.target | e}} {% endif %} {% if invocation.build_wall_clock_time %} {{ invocation.build_wall_clock_time | e}} {% endif %} {% if invocation.exit_code != 0 %} {% if invocation.exit_code == 1 %} build failed {% elif invocation.exit_code == 2 %} failed {% elif invocation.exit_code == 8 %} analysis failure {% elif invocation.exit_code == 16 %} build-environment failure {% elif invocation.exit_code == 32 %} syntax/invocation error {% else %} exit code {{ invocation.exit_code | e }} {% endif %} {% endif %}
      {% if invocation.remote_address %}
    • Remote: {{ invocation.remote_address | e }}@{{ invocation.remote_props | e }}
      • {% endif %} {% if invocation.config %}
    • Configuration: {{ invocation.config | e}}
      • {% endif %} {% if invocation.context %}
    • Context: {{ invocation.context | e }}
      • {% endif %}
    • {% endfor %} {% if reduced_invocations %}
  • {{ reduced_invocations_count | e }} more
      {% for invocation in reduced_invocations %}
    • {{ invocation.name | e }} {% if invocation.context %} Context: {{ invocation.context | e }} {% endif %}
      • {% endfor %}
    • {% endif %}
{% else %} (none) {% endif %} {% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/macros.html000066400000000000000000000053051516554100600313540ustar00rootroot00000000000000{% macro show_action(action, first = false) %}
  • {% if first %}
    {% else %}
    {% endif %} {{ action["name_prefix"] | e }} {{ action["name"] | e}} {% if action["may_fail"] %} {% if action["exit_code"] != 0 %} failure: {{ action["may_fail"] }} {% else %} tainted ({{ action["may_fail"] }}) {% endif %} {% elif action["primary_output"] %} {% if action["exit_code"] != 0 %} failed to build: {{ action["primary_output"] | e }} {% else %} build: {{ action["primary_output"] | e }} {% endif %} {% endif %}
      {% if action["stdout"] %}
    • stdout: {{action["stdout"] | e }}
      • {% endif %} {% if action["stderr"] %}
    • stderr: {{action["stderr"] | e }}
      • {% endif %} {% if action["cmd"] %}
    • command {{ action["cmd"] | e }}
      • {% endif %} {% if action["origins"] %}
    • origins
        {% for origin in action["origins"] %}
      • {{ origin | e }}
        • {% endfor %}
      • {% endif %} {% if action["duration"] %}
    • duration: {{ action["duration"] | e }}
      • {% endif %} {% if action["cached"] %}
    • cached: {{ action["cached"] | e }}
      • {% endif %} {% if action["exit_code"] != 0 %}
    • exit code: {{ action["exit_code"] }}
      • {% endif %} {% if action["output"] %}
    • output
        {% for out in action["output"] %} {% if action["artifacts"].get(out) %}
      • {{ out | e }} [↓]
        • {% else %}
      • {{ out | e }}
        • {% endif %} {% endfor %}
      • {% endif %} {% if action["output_dirs"] %}
    • output directories
        {% for out in action["output_dirs"] %} {% if action["artifacts"].get(out) %}
      • {{ out | e }}
        • {% else %}
      • {{ out | e }}
        • {% endif %} {% endfor %}
      • {% endif %}
    • {% endmacro %} just-buildsystem-justbuild-b1fb5fa/doc/invocations-http-server/templates/tree.html000066400000000000000000000006651516554100600310330ustar00rootroot00000000000000{% extends "base.html" %} {% block heading %} Tree {{ name | e }} {% endblock %} {% block content %}

    Tree {{ name | e }}

    {% endblock %} just-buildsystem-justbuild-b1fb5fa/doc/specification/000077500000000000000000000000001516554100600232045ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/specification/remote-protocol.md000066400000000000000000000154161516554100600266670ustar00rootroot00000000000000Specification of the just Remote Execution Protocol =================================================== Introduction ------------ just supports remote execution of actions across multiple machines. As such, it makes use of a remote execution protocol. The basis of our protocol is the open-source gRPC [remote execution API](https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto). We use this protocol in a **compatible** mode, but by default, we use a modified version, allowing us to pass git trees and files directly without even looking at their content or traversing them. This modification makes sense since it is more efficient if sources are available in git repositories and much open-source code is hosted in git repositories. With this protocol, we take advantage of already hashed git content as much as possible by avoiding unnecessary conversion and communication overhead. In the following sections, we explain which modifications we applied to the original protocol and which requirements we have to the remote execution service to seamlessly work with just. just Protocol Description ------------------------- ### git Blob and Tree Hashes In order to be able work with git hashes, both client side as well as server side need to be extended to support the regular git hash functions for blobs and trees: The hash of a blob is computed as sha1sum(b"blob \0") The hash of a tree is computed as sha1sum(b"tree \0") where `` is a sequence (without newlines) of ``, and each `` is \0 `` is a number defining if the object is a file (`100644`), an executable file (`100755`), a tree (`040000`), or a symbolic link (`120000`). More information on how git internally stores its objects can be found in the official [git documentation](https://git-scm.com/book/en/v2/git-Internals-git-Objects). Since git hashes blob content differently from trees, this type of information has to be transmitted in addition to the content and the hash. To this aim, just prefixes the git hash values passed over the wire with a single-byte marker. Thus allowing the remote side to distinguish a blob from a tree without inspecting the (potentially large) content. The markers are - `0x62` for a git blob (`0x62` corresponds to the character `b`) - `0x74` for a git tree (`0x74` corresponds to the character `t`) Since hashes are transmitted as hexadecimal string, the resulting length of such prefixed git hashes is 42 characters. The server side has to accept this hash length as valid hash length to detect our protocol and to apply the according git hash functions based on the detected prefix. ### Blob and Tree Availability Typically, it makes sense for a client to check the availability of a blob or a tree at the remote side, before it actually uploads it. Thus, the remote side should be able to answer availability requests based on our prefixed hash values. ### Blob Upload A blob is uploaded to the remote side by passing its raw content as well as its `Digest` containing the git hash value for a blob prefixed by `0x62`. The remote side needs to verify the received content by applying the git blob hash function to it, before the blob is stored in the content addressable storage (CAS). If a blob is part of git repository and already known to the remote side, we even do not have to calculate the hash value from a possible large file, instead we can directly use the hash value calculated by git and pass it through. ### Tree Upload In contrast to regular files, which are uploaded as blobs, the original protocol has no notion of directories on the remote side. Thus, directories need to be traversed and converted to `Directory` Protobuf messages, which are then serialized and uploaded as blobs. In our modified protocol, we prevent this traversing and conversion overhead by directly uploading the git tree objects instead of the serialized Protobuf messages if the directory is part of a git repository. Consequently, we can also reuse the corresponding git hash value for a tree object, which just needs to be prefixed by `74`, when uploaded. The remote side must accepts git tree objects instead `Directory` Protobuf messages at any location where `Directory` messages are referred (e.g., the root directory of an action). The tree content is verified using the git hash function for trees. In addition, it has to be modified to parse the git tree object format. Using this git tree representation makes tree handling much more efficient, since the effort of traversing and uploading the content of a git tree occurs only once and for each subsequent request, we directly pass around the git tree id. We require the invariant that if a tree is part of any CAS then all its content is also available in this CAS. To adhere to this invariant, the client side has to prove that the content of a tree is available in the CAS, before uploading this tree. One way to ensure that the tree content is known to the remote side is that it is uploaded by the client. The server side has to ensure this invariant holds. In particular, if the remote side implements any sort of pruning strategy for the CAS, it has to honor this invariant when an element got pruned. Another consequence of this efficient tree handling is that it improves **action digest** calculation noticeably, since known git trees referred by the root directory do not need to be traversed. This in turn allows to faster determine whether an action result is already available in the action cache or not. ### Tree Download Once an action is successfully executed, it might have generated output files or output directories in its staging area on the remote side. Each output file needs to be uploaded to its CAS with the corresponding git blob hash. Each output directory needs to be translated to a git tree object and uploaded to the CAS with the corresponding git tree hash. Only if the content of a tree is available in the CAS, the server side is allowed to return the tree to the client. In case of a generated output directory, the server only returns the corresponding git tree id to the client instead of a flat list of all recursively generated output directories as part of a `Tree` Protobuf message as it is done in the original protocol. The remote side promises that each blob and subtree contained in the root tree is available in the remote CAS. Such blobs and trees must be accessible, using the streaming interface, without specifying the size (since sizes are not stored in a git tree). Due to the [Protobuf 3 specification](https://protobuf.dev/reference/protobuf/proto3-spec/), which is used in this remote execution API, not specifying the size means the default value `0` is used. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/000077500000000000000000000000001516554100600222275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/doc/tutorial/build-delegation.md000066400000000000000000000074271516554100600257730ustar00rootroot00000000000000# More build delegation using a serve endpoint The original purpose of a `just serve` endpoint is to allow building against dependencies without having to download them. That is particularly important when [bootstrapping](https://bootstrappable.org/) the toolchain. However, the serve endpoint does not care what the target actually is. As long as it is a content-fixed `export` target, it has all the necessary roots. Therefore, it can also be used for other purposes. ## Example: Analysing large data sets Besides sources of long bootstrap chains, all form of measurement data are also files that one wants to avoid having to download, while still analysing them in various ways and by several persons. ### Making data available to serve Depending on the nature of the data set to be analysed, several ways are appropriate to make it available to serve. Data for long-term archival, such as experimental measurements, can be committed to a repository and that repository added to the `"repositories"` field in the serve configuration as usual. There is, however, another possibility more suited for data to be rotated, like monitoring data or invocation-log data written by `just-mr`. Each entity generating such data (like monitoring machine, CI runner, etc.) uploads the data directory to the remote-execution endpoint, e.g., via `just-mr add-to-cas` and only distributes the tree hash to the entities analysing the data. As a user of the serve endpoint, by just knowing the tree hash, can construct an absent root from it. ``` jsonc { "repository": { "type": "git tree" , "id": "..." , "cmd": ["false", "Should be known to CAS"] , "pragma": {"absent": true} } } ``` Of course, the command `false` is not able to create the specified tree, but it should not be executed anyway, especially as we don't want to ever have that large tree locally. Buildings against this root still makes it available to serve without ever fetching it; the reason this works is that `just-mr` always prefers the network-wise closest path: if the root is not known to the serve endpoint anyway, but is known to the remote-execution CAS, it simply asks serve to fetch it from there. No need to get the root local, as it is marked absent. Of course, the above root description is so systematic, that we can easily generate it from the hash; this is useful if we have many data sets uploaded individually and hence need many of those repositories. ### Analysing data via serve To analyse a data set, we need, besides the actual data, also a target description and, potentially, additional tools. Here we use that `just` allows separate layers for sources and targets. So we can add a separate repository with the targets file for analysing the data. As that one will typically be small, we can write it locally (allowing us the experiment with different kinds of statistics we might care about) and mark it as `"to_git"`. This not only makes it content-fixed, but also ensures that it will be uploaded to the serve endpoint. For computations delegated to serve, we can only access export targets; but while measurement data might have some random component, analysing that data typically is a pure function. So a simple target file could look as follows. ``` jsonc { "": {"type": "export", "target": "stats"} , "stats": { "type": "generic" , "outs": ["stats.json"] , "cmds": ["./statistics-tool"] , "deps": ["data", ["@", "tools", "", "statistics-tool"]] } , "data": {"type": "install", "dirs": [[["TREE", null, "."], "data"]]} } ``` If the data tree contains several data sets that can be analysed independently, instead of using a big action, several tasks can be defined using computed roots. If many different data trees are uploaded, an overall accumulation of the data of the individual repositories can be carried out. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/computed.md000066400000000000000000000311521516554100600243730ustar00rootroot00000000000000# Computed Roots The general approach of writing a build description side-by-side with the source code works in most cases. There are, however, cases where the build description depends on the contents of source-like files. Here we consider a somewhat contrived example that, however, shows all the various types of derived roots. Let's say we have a very regular structure of our code base: one top-level directory for each library and if there are dependencies, then there is a plain file `deps` listing, one entry per line, the libraries depended upon. From that structure we want a derived build description that is not maintained manually. As an example, say, so far we have the file structure ``` src +--foo | +-- foo.hpp | +-- foo.cpp | +--bar +-- bar.hpp +-- bar.cpp +-- deps ``` where `src/bar/deps` contains a single line, saying `foo`. The first step is to write a generator for a single `TARGETS` file. To clearly separate the infrastructure files from the sources, we add the generator as `utils/generate.py`. ```{.python srcname="utils/generate.py"} #!/usr/bin/env python3 import json import sys name = sys.argv[1] deps = [] if len(sys.argv) > 2: with open(sys.argv[2]) as f: deps = f.read().splitlines() target = {"type": ["@", "rules", "CC", "library"], "name": [name], "hdrs": [["GLOB", None, "*.hpp"]], "srcs": [["GLOB", None, "*.cpp"]], "stage": [name], } if deps: target["deps"] = [[x, ""] for x in deps] targets = {"": target} with open("TARGETS", "w") as f: json.dump(targets, f) f.write("\n") ``` A `TARGETS` file has to be created for every directory containing files (and not just other directories). Additionally, there needs to be a top-level target staging all those files that is exported. This can be implemented by another script, say `utils/call-generator-targets.py`. ```{.python srcname="utils/call-generator-targets.py"} #!/usr/bin/env python3 import json import sys import os targets = {} stage = {} for root, dirs, files in os.walk("."): if files: target_name = "lib " + root with_deps = "deps" in files deps_name = os.path.join(root, "deps") entry = {"type": "generic", "outs": ["TARGETS"], "deps": ([["@", "utils", "", "generate.py"]] + ([["", deps_name]] if with_deps else [])), "cmds": ["./generate.py " + os.path.normpath(root) + (" " + deps_name if with_deps else "")]} targets[target_name] = entry stage[os.path.normpath(os.path.join(root, "TARGETS"))] = target_name targets["stage"] = {"type": "install", "files": stage} targets[""] = {"type": "export", "target": "stage"} with open(sys.argv[1], "w") as f: json.dump(targets, f, sort_keys=True) f.write("\n") ``` Of course, those scripts have to be executable. ```shell $ chmod 755 utils/*.py ``` With that, we can generate the build description for generating the target files. We first write a target file `utils/targets.generate`. ```{.json srcname="utils/targets.generate"} { "": {"type": "export", "target": "generate"} , "generate": { "type": "generic" , "cmds": ["cd src && ../call-generator-targets.py ../TARGETS"] , "outs": ["TARGETS"] , "deps": [["@", "utils", "", "call-generator-targets.py"], "src"] } , "src": {"type": "install", "dirs": [[["TREE", null, "."], "src"]]} } ``` As we intend to make `utils` a separate logical repository, we also add a trivial top-level targets file. ```shell $ echo {} > utils/TARGETS ``` Next we can start a repository description. Here we notice that the tasks to be performed to generate the target files only depend on the tree structure of the `src` repository. So, we use the tree structure as workspace root to avoid unnecessary runs of `utils/targets.generate`. ```{.json srcname="etc/repos.json"} { "repositories": { "src": {"repository": {"type": "file", "path": "src", "pragma": {"to_git": true}}} , "utils": { "repository": {"type": "file", "path": "utils", "pragma": {"to_git": true}} } , "src target tasks description": { "repository": {"type": "tree structure", "repo": "src"} , "target_root": "utils" , "target_file_name": "targets.generate" , "bindings": {"utils": "utils"} } } } ``` Of course, the `"to_git"` pragma works best, if we have everything under version control (which is a good idea in general anyway). ```shell $ git init $ git add . $ git commit -m 'Initial commit' ``` Now the default target of `"src target tasks description"` shows how to build the target files we want. ```shell $ just-mr --main 'src target tasks description' build -p INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-p"] INFO: Repository "src target tasks description" depends on 1 top-level computed roots INFO: Requested target is [["@","src target tasks description","",""],{}] INFO: Analysed target [["@","src target tasks description","",""],{}] INFO: Export targets found: 0 cached, 1 uncached, 0 not eligible for caching INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","src target tasks description","",""],{}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: TARGETS [68336b9823a86d0f64a5a79990c6b171d4f6523b:434:f] {"": {"target": "stage", "type": "export"}, "lib ./bar": {"cmds": ["./generate.py bar ./bar/deps"], "deps": [["@", "utils", "", "generate.py"], ["", "./bar/deps"]], "outs": ["TARGETS"], "type": "generic"}, "lib ./foo": {"cmds": ["./generate.py foo"], "deps": [["@", "utils", "", "generate.py"]], "outs": ["TARGETS"], "type": "generic"}, "stage": {"files": {"bar/TARGETS": "lib ./bar", "foo/TARGETS": "lib ./foo"}, "type": "install"}} INFO: Backing up artifacts of 1 export targets ``` From that, we can, step by step, define the actual build description. - The tasks to generate the target files is a computed root of the `"src target tasks"` using the top-level target `["", ""]`. We call it `"src target tasks"`. - This root will be the target root in the repository `src target build` describing how to generate the actual target files. - The `"src targets"` are then, again, a computed root. - Finally, we can define the top-level repository `""`. As we want to be able to build with uncommitted changes as long as they do not affect the target description, we use an explicit file repository instead of referring to the `to_git` repository `"src"`. - As the top-level targets also depend on our C/C++ rules, we include those as well and set an appropriate binding for `""`. Therefore, our final repository description looks as follows. ```{.json srcname="etc/repos.json"} { "repositories": { "src": {"repository": {"type": "file", "path": "src", "pragma": {"to_git": true}}} , "utils": { "repository": {"type": "file", "path": "utils", "pragma": {"to_git": true}} } , "src target tasks description": { "repository": {"type": "tree structure", "repo": "src"} , "target_root": "utils" , "target_file_name": "targets.generate" , "bindings": {"utils": "utils"} } , "src target tasks": { "repository": { "type": "computed" , "repo": "src target tasks description" , "target": ["", ""] } } , "src target build": { "repository": "src" , "target_root": "src target tasks" , "bindings": {"utils": "utils"} } , "src targets": { "repository": {"type": "computed", "repo": "src target build", "target": ["", ""]} } , "": { "repository": {"type": "file", "path": "src"} , "target_root": "src targets" , "bindings": {"rules": "rules"} } , "rules": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } } } } ``` With that, we can now analyse `["bar", ""]` and see that the dependency we wrote in `src/bar/deps` is honored. With increased log level we can also see hints on the computation of the computed roots. ```shell $ just-mr analyse --log-limit 4 bar '' INFO: Performing repositories setup INFO: Found 8 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","--log-limit","4","bar",""] INFO: Repository "" depends on 1 top-level computed roots PERF: Export target ["@","src target tasks description","",""] taken from cache: [52dd0203238644382280dc9ae79c75d1f7e5adf1:120:f] -> [79a4114597a03d82acfd95a5f95f0d22c6f09ccb:582:f] PERF: Root [["@","src target tasks description","",""],{}] evaluated to e5dfc10a073e3e101a256bc38fae67ec234afccb, log aa803f1f445bfe20826495e33252ea02c4c1d7e0 PERF: Export target ["@","src target build","",""] registered for caching: [309aac8800c83359aa900b7368b25b03bb343110:120:f] PERF: Root [["@","src target build","",""],{}] evaluated to db732bc9b76cb485970795dad3de7941567f4caa, log c33eb0167ccd7b5b3b8db51657d0d80885c61f98 INFO: Requested target is [["@","","bar",""],{}] INFO: Analysed target [["@","","bar",""],{}] INFO: Result of target [["@","","bar",""],{}]: { "artifacts": { "bar/libbar.a": {"data":{"id":"081122c668771bb09ef30b12687c6f131583506714a992595133ab9983366ce7","path":"work/bar/libbar.a"},"type":"ACTION"} }, "provides": { "compile-args": [ ], "compile-deps": { "foo/foo.hpp": {"data":{"path":"foo/foo.hpp","repository":""},"type":"LOCAL"} }, "debug-hdrs": { }, "debug-srcs": { }, "dwarf-pkg": { }, "link-args": [ "bar/libbar.a", "foo/libfoo.a" ], "link-deps": { "foo/libfoo.a": {"data":{"id":"613a6756639b7fac44a698379581f7ac9113536f95722e4180cae3af45befeb9","path":"work/foo/libfoo.a"},"type":"ACTION"} }, "lint": [ ], "package": { "cflags-files": {}, "ldflags-files": {}, "name": "bar" }, "run-libs": { }, "run-libs-args": [ ] }, "runfiles": { "bar/bar.hpp": {"data":{"path":"bar/bar.hpp","repository":""},"type":"LOCAL"} } } ``` The quoted logs can be inspected with the `install-cas` subcommand as usual. To see how target files adapt to source changes, let's add a new directory `baz` with source and header files, as well as a `deps` file saying `bar`. ```shell $ mkdir src/baz $ echo '#include "bar/bar.hpp" #...' > src/baz/baz.hpp $ touch src/baz/baz.cpp $ echo 'bar' > src/baz/deps ``` As this affects the target structure, we commit those changes. ```shell $ git add . && git commit -m 'New library baz' ``` After that, we can immediately build the new library. ```shell $ just-mr build --log-limit 4 baz '' INFO: Performing repositories setup INFO: Found 8 repositories involved INFO: Setup finished, exec ["just","build","-C","...","--log-limit","4","baz",""] INFO: Repository "" depends on 1 top-level computed roots PERF: Export target ["@","src target tasks description","",""] registered for caching: [02e0545e14758f7fe08a90b56cbfae2e12bdd51e:120:f] PERF: Root [["@","src target tasks description","",""],{}] evaluated to 43a0b068d6065519061b508a22725c50e68279be, log bf4cc2d0d803bcff78bd7e4e835440467f3a3674 PERF: Export target ["@","src target build","",""] registered for caching: [00c234393fc6986c308c16d6847ed09e79282097:120:f] PERF: Root [["@","src target build","",""],{}] evaluated to 739a4750d43328ad9c6ff7d9445246a6506368fd, log 51525391c622a398b5190adee07c9de6338d56ba INFO: Requested target is [["@","","baz",""],{}] INFO: Analysed target [["@","","baz",""],{}] INFO: Discovered 6 actions, 0 tree overlays, 3 trees, 0 blobs INFO: Building [["@","","baz",""],{}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: baz/libbaz.a [c6eb3219ec0b1017f242889327f9c2f93a316546:1060:f] (1 runfiles omitted.) INFO: Target tainted ["test"]. ``` Obviously, the tree structure has changed, so `"src target tasks description"` target gets rebuild. Also, the `"src target build"` target gets rebuild, but if we inspect the log, we see that 2 out of 3 actions are taken from cache. A similar construction is also used in the `justbuild` main `git` repository for describing the task of formatting all JSON files: the target root of the logical repository `"format-json"` is computed, based on the underlying tree structure. Again, the workspace root for `"format-json"` is the plain file root, so that uncommitted changes (to committed files) are taken into account. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/cross-compiling.md000066400000000000000000000511601516554100600256640ustar00rootroot00000000000000Cross compiling and testing cross-compiled targets ================================================== So far, we were always building for the platform on which we were building. There are, however, good reasons to build on a different one. For example, the other platform could be faster (common theme when developing for embedded devices), cheaper, or simply available in larger quantities. The standard solution for these kind of situations is cross compiling: the binary is completely built on one platform, while being intended to run on a different one. Cross compiling using the CC rules ---------------------------------- The `C` and `C++` rules that come with the `justbuild` repository already have cross-compiling already built into the toolchain definitions for the `"gnu"` and `"clang"` compiler families; as different compilers expect different ways to be told about the target architecture, cross compiling is not supported for the `"unknown"` compiler family. First, ensure that the required tools and libraries (which also have to be available for the target architecture) for cross compiling are installed. Depending on the distribution used, this can be spread over several packages, often shared by the architecture to cross compile to. Once the required packages are installed, usage is quite forward. Let's start a new project in a clean working directory. ``` sh $ touch ROOT ``` We create a file `repos.template.json` specifying the one local repository. ``` {.jsonc srcname="repos.template.json"} { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc"} } } } ``` The actual `rules-cc` are imported via `just-import-git`. ``` sh $ just-import-git -C repos.template.json -b master --as rules-cc https://github.com/just-buildsystem/rules-cc > repos.json ``` Let's have `main.cpp` be the canonical _Hello World_ program. ``` {.cpp srcname="main.cpp"} #include int main() { std::cout << "Hello world!\n"; return 0; } ``` Then, a `TARGETS` file describing a simple binary. ``` {.jsonc srcname="TARGETS"} { "helloworld": { "type": ["@", "rules", "CC", "binary"] , "name": ["helloworld"] , "srcs": ["main.cpp"] } } ``` As mentioned in the introduction, we need to specify `TOOLCHAIN_CONFIG`, `OS`, and `ARCH`. So the canonical building for host looks something like the following. ``` sh $ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64"}' INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\"}"] INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [0d5754a83c7c787b1c4dd717c8588ecef203fb72:16992:x] $ ``` To cross compile, we simply add `TARGET_ARCH`. ``` sh $ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [b45459ea3dd36c7531756a4de9aaefd6af30e417:9856:x] $ ``` To inspect the different command lines for native and cross compilation, we can use `just analyse`. ``` sh $ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64"}' --dump-actions - $ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - $ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "clang"}, "OS": "linux", "ARCH": "x86_64"}' --dump-actions - $ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "clang"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - ``` Testing in the presence of cross compiling ------------------------------------------ To understand testing the in the presence of cross compiling, let's walk through a simple example. We create a basic test in subdirectory `test` ``` sh $ mkdir -p ./test ``` by adding a file `test/TARGETS` containing ``` {.jsonc srcname="test/TARGETS"} { "basic": { "type": ["@", "rules", "shell/test", "script"] , "name": ["basic"] , "test": ["basic.sh"] , "deps": [["", "helloworld"]] } , "basic.sh": { "type": "file_gen" , "name": "basic.sh" , "data": "./helloworld | grep -i world" } } ``` Now, if we try to run the test by simply specifying the target architecture, we find that the binary to be tested is still only built for host. ``` sh $ just-mr analyse --dump-targets - -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' test basic INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","--dump-targets","-","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}","test","basic"] INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Analysed target [["@","","test","basic"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Result of target [["@","","test","basic"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]: { "artifacts": { "pwd": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"pwd"},"type":"ACTION"}, "result": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"result"},"type":"ACTION"}, "stderr": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"stderr"},"type":"ACTION"}, "stdout": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"stdout"},"type":"ACTION"}, "time-start": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"time-start"},"type":"ACTION"}, "time-stop": {"data":{"id":"eede84ac9ccc613b4542d93b321fc2ef9dc7f260d49708e6512518254a6c0237","path":"time-stop"},"type":"ACTION"} }, "provides": { "lint": [ ] }, "runfiles": { "basic": {"data":{"id":"e5efe00530b6d0586a4d4257b01e5f9655958df30f5ec6db86d709f195078f29"},"type":"TREE"} } } INFO: List of analysed targets: { "@": { "": { "": { "helloworld": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] }, "test": { "basic": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","ARCH_DISPATCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":"linux","RUNS_PER_TEST":null,"TARGET_ARCH":"arm64","TEST_ENV":null,"TEST_SUMMARY_EXECUTION_PROPERTIES":null,"TIMEOUT_SCALE":null,"TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}], "basic.sh": [{}] } }, "rules-cc": { "CC": { "defaults": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] }, "shell": { "defaults": [{"ARCH":"x86_64","HOST_ARCH":null,"TARGET_ARCH":"arm64"}] } }, "rules-cc/just/rules": { "CC": { "defaults": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] } }, "rules-cc/just/toolchain": { "CC": { "defaults": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}], "gcc": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}] } } } } INFO: Target tainted ["test"]. $ ``` The reason is that a test actually has to run the created binary and that requires a build environment of the target architecture. So, if not being told how to obtain such an environment, they carry out the test in the best manner they can, i.e., by transitioning everything to host. So, in order to properly test the cross-compiled binary, we need to do two things. - We need to setup remote execution on the correct architecture, either by buying the appropriate hardware, or by running an emulator. - We need to tell *justbuild* on how to reach that endpoint. To continue the example, let's say we set up an `arm64` machine, e.g., a Raspberry Pi, in the local network. On that machine, we can simply run a single-node execution service using `just execute`; note that the `just` binary used there has to be an `arm64` binary, e.g., obtained by cross compiling. The next step is to tell *justbuild* how to reach that machine; as we only want to use it for certain actions we can't simply set it as (default) remote-execution endpoint (specified by the `-r` option). Instead we create a file `dispatch.json`. ``` {.jsonc srcname="dispatch.json"} [[{"runner": "arm64-worker"}, "10.204.62.67:9999"]] ``` This file contains a list of lists pairs (lists of length 2) with the first entry a map of remote-execution properties and the second entry a remote-execution endpoint. For each action, if the remote-execution properties of that action match the first component of a pair the second component of the first matching pair is taken as remote-execution endpoint instead of the default endpoint specified on the command line. Here "matching" means that all values in the domain of the map have the same value in the remote-execution properties of the action (in particular, `{}` matches everything); so more specific dispatches have to be specified earlier in the list. In our case, we have a single endpoint in our private network that we should use whenever the property `"runner"` has value `"arm64-worker"`. The IP/port specification might differ in your setup. The path to this file is passed by the `--endpoint-configuration` option. Next, we have to tell the test rule, that we actually do have a runner for the `arm64` architecture. The rule expects this in the `ARCH_DISPATCH` configuration variable that you might have seen in the list of analysed targets earlier. It is expected to be a map from architectures to remote-execution properties ensuring the action will be run on that architecture. If the rules finds a non-empty entry for the specified target architecture, it assumes a runner is available and will run the test action with those properties; for the compilation steps, still cross compiling is used. In our case, we set `ARCH_DISPATCH` to `{"arm64": {"runner": "arm64-worker"}}`. Finally, we have to provide the credentials needed for mutual authentication with the remote-execution endpoint by setting `--tls-*` options appropriately. `just` assumes that the same credentials can be used all remote-execution endpoints involved. In our example we're building locally (where the build process starts the actions itself) which does not require any credentials; nevertheless, it still accepts any credentials provided. ``` sh $ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64", "ARCH_DISPATCH": {"arm64": {"runner": "arm64-worker"}}}' --endpoint-configuration dispatch.json --tls-ca-cert ca.crt --tls-client-cert client.crt --tls-client-key client.key test basic INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\", \"ARCH_DISPATCH\": {\"arm64\": {\"runner\": \"arm64-worker\"}}}","--endpoint-configuration","dispatch.json","--tls-ca-cert","ca.crt","--tls-client-cert","client.crt","--tls-client-key","client.key","test","basic"] INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Analysed target [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Target tainted ["test"]. INFO: Discovered 3 actions, 0 tree overlays, 3 trees, 3 blobs INFO: Building [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 3 actions, 3 cache hits. INFO: Artifacts built, logical paths are: pwd [da762809b16a6b476c6d45f67695769bfd6fdb01:160:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] stderr [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] stdout [cd0875583aabe89ee197ea133980a9085d08e497:13:f] time-start [32fdb8269cd3a5a699eeeafd1be565308167c110:11:f] time-stop [32fdb8269cd3a5a699eeeafd1be565308167c110:11:f] (1 runfiles omitted.) INFO: Target tainted ["test"]. $ ``` The resulting command line might look complicated, but the authentication-related options, as well as the dispatch-related options (including setting `ARGUMENTS_DISPATCH` via `-D`) can simply be set in the `"just args"` entry of the `.just-mrrc` file. When inspecting the result, we can use `just install-cas` as usual, without any special arguments. Whenever dispatching an action to a non-default endpoint, `just` will take care of syncing back the artifacts to the default CAS. Testing a matrix of configurations ---------------------------------- This example also shows that a single test can be run in a variety of configurations. - Different compilers can be used. - We can build and test for the host architecture, or cross-compile for a different architecture and run the tests there. Often, all possible combinations need to be tested, especially for projects that are meant to be portable. Doing this by hand or manually maintaining all combinations as a list of CI tasks is cumbersome and error prone. Therefore, there is a rule `["@", "rules", "test", "matrix"]` that can take care of handling all configuration combinations in a single build. The fields are the same as of a test suite. ``` {.jsonc srcname="test/TARGETS"} ... , "matrix": { "type": ["@", "rules", "test", "matrix"] , "deps": ["basic"] } ... ``` If run without special configuration, it also behaves like a test suite. ``` shell $ just-mr build test matrix INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","build","-C","...","test","matrix"] INFO: Requested target is [["@","","test","matrix"],{}] INFO: Analysed target [["@","","test","matrix"],{}] INFO: Target tainted ["test"]. INFO: Discovered 3 actions, 0 tree overlays, 3 trees, 3 blobs INFO: Building [["@","","test","matrix"],{}]. INFO: Processed 3 actions, 0 cache hits. INFO: Artifacts built, logical paths are: basic [4a75cd3ff7b5b6552c03d74580c1d336f3f38a4e:208:t] INFO: Target tainted ["test"]. ``` But we can instruct it via the configuration variable `TEST_MATRIX` to run its `"deps"` in a product of configurations, with `TEST_MATRIX` cleared there. ``` shell $ just-mr describe test matrix INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","describe","-C","...","test","matrix"] [["@","","test","matrix"],{}] is defined by user-defined rule ["@","rules-cc","test","matrix"]. | Given a group of tests, build them in a variety of configurations. | | The configuration variable TEST_MATRIX is expected to be a map with | each value being a map itself. Sequentially for each key, all possible | values of the associated map are tried and staged to the appropriate | key. Thus, the tests in "deps" are built in an exponential number of | configurations. | | If TEST_MATRIX is unset, {} will be assumed, i.e., all the "deps" will | be built precisely once, in the current configuration. In this way, | the "matrix" rule can be used instead of the "suite" rule to allow | user-defined configuration matrices, dispatching over parameters for | dependencies (e.g., the toolchain). String fields - "stage" | The logical location this test suite is to be placed. | Individual entries will be joined with "/". Target fields - "deps" | The targets that suite is composed of. Variables taken from the configuration - "TEST_MATRIX" | Map describing the dimensions of the matrix to run the tests for. | | Keys are config variables. The value for each key has to be a map | mapping the stage name to the corresponding value for that config | variable. Result - Artifacts | The disjoint union of the runfiles of the "deps" targets, | evaluated and staged as requested by TEST_MATRIX and finally | staged to the location given by "stage". - Runfiles | Same as artifacts. ``` Typically, one sets the matrix of desired configurations in a `"configure"` target next to the matrix target which then acts as the main entry point. ``` {.jsonc srcname="test/TARGETS"} ... , "ALL": { "type": "configure" , "tainted": ["test"] , "target": "matrix" , "config": { "type": "'" , "$1": { "TEST_MATRIX": { "TARGET_ARCH": {"x86": "x86_64", "arm": "arm64"} , "TOOLCHAIN_CONFIG": {"gcc": {"FAMILY": "gnu"}, "clang": {"FAMILY": "clang"}} } } } } ... ``` Each key in `"TEST_MATRIX"` is a configuration variable where different values should be taken into account. For each such variable we specify in another map the values that should be tried, keyed by the path there the tests should be staged; obviously, if we run the same test in many configurations we need to stage the various test results to different locations. Now, in a single invocation, we can run our test in all four relevant configurations. Still, credentials and dispatch information needs to be provided. ``` shell $ just-mr build -D '{"OS": "linux", "ARCH": "x86_64", "ARCH_DISPATCH": {"arm64": {"runner": "arm64-worker"}}}' --endpoint-configuration dispatch.json --tls-ca-cert ca.crt --tls-client-cert client.crt --tls-client-key client.key test ALL INFO: Performing repositories setup INFO: Found 22 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"OS\": \"linux\", \"ARCH\": \"x86_64\", \"ARCH_DISPATCH\": {\"arm64\": {\"runner\": \"arm64-worker\"}}}","--endpoint-configuration","dispatch.json","--tls-ca-cert","ca.crt","--tls-client-cert","client.crt","--tls-client-key","client.key","test","ALL"] INFO: Requested target is [["@","","test","ALL"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux"}] INFO: Analysed target [["@","","test","ALL"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux"}] INFO: Target tainted ["test"]. INFO: Discovered 12 actions, 0 tree overlays, 9 trees, 3 blobs INFO: Building [["@","","test","ALL"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux"}]. INFO: Processed 12 actions, 6 cache hits. INFO: Artifacts built, logical paths are: clang/arm/basic [6503faa5935250fe50ebc2f00dbdd9dbf592fd3a:208:t] clang/x86/basic [90ead7c3b83f6902c3b0b032d63d734e0ea55a0b:208:t] gcc/arm/basic [24c3f2de7e920ad294edba63da596d810a2c350a:208:t] gcc/x86/basic [4a75cd3ff7b5b6552c03d74580c1d336f3f38a4e:208:t] INFO: Target tainted ["test"]. ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/debugging.md000066400000000000000000000132461516554100600245120ustar00rootroot00000000000000Debugging ========= With *justbuild* striving to ensure that only our own project's code is needed to be available locally, and not any of our dependencies (especially in conjunction with our [serve service](./just-serve.md)) or any on-the-fly patches or generated files, it might seem that debugging is not a straight-forward endeavor. However, *justbuild* always knows exactly what source files are needed to build a target and therefore they can be staged locally and made available to a debugger, such as `gdb(1)`. In the following we will show how to use the `["CC", "install-with-deps"]` rule to stage all the needed sources for debugging a program with `gdb(1)`. As example we use the *hello_world* program employing high-level target caching from the section on [*Building Third-party dependencies*](./third-party-software.md), which depends on the open-source project [fmtlib](https://github.com/fmtlib/fmt). The debugging process --------------------- The first step is to define a set of debugging defaults (toolchain and compile flags) for our example project. This can be done by extending the existing `tutorial-defaults/CC/TARGETS` file as ``` {.jsonc srcname="tutorial-defaults/CC/TARGETS"} { "defaults": { "type": ["CC", "defaults"] , "CC": ["cc"] , "CXX": ["c++"] , "ADD_COMPILE_FLAGS": ["-O2", "-Wall"] , "ADD_DEBUGFLAGS": ["-O2", "-Wall", "-g"] , "AR": ["ar"] , "DWP": ["dwp"] , "PATH": ["/bin", "/usr/bin"] } } ``` This states that when in debug mode a different set of C/C++ flags should be used, in this case the ones from release mode plus the `-g` flag to generate debug information for `gdb(1)`. As this updated `TARGETS` file is under version control, we need to commit the changes: ``` sh $ git add tutorial-defaults $ git commit -m "update compile flags for debugging" [master 642f739] update compile flags for debugging 1 file changed, 1 insertions(+) ``` Now we need to configure the actual target we want to debug. The `TARGETS` file of our *hello-world* program currently contains only the release version, so we need to add a new target, configured in debug mode. ``` {.jsonc srcname="TARGETS"} ... , "helloworld-debug": { "type": "configure" , "target": "helloworld" , "config": {"type": "'", "$1": {"DEBUG": {"USE_DEBUG_FISSION": false}}} } ... ``` This describes the debug version of *hello-world*, configured by setting the `"DEBUG"` map to enable regular (non-fission) debug mode, which in turn will instruct the updated toolchain defaults, which honor the `"DEBUG"` argument, to compile all actions with the `"-g"` flag. To actually collect the artifacts needed to run the debugger, we use the `["CC", "install-with-deps"]` rule, contained in the `rules-cc` repository. ``` {.jsonc srcname="TARGETS"} ... , "helloworld-debug staged": { "type": ["@", "rules", "CC", "install-with-deps"] , "targets": ["helloworld-debug"] } ... ``` Now this target can be installed to a location of our choice provided by the `-o` argument. ``` sh $ just-mr install "helloworld-debug staged" -o .ext/debug INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","install","-C","...","helloworld-debug staged","-o",".ext/debug"] INFO: Requested target is [["@","tutorial","","helloworld-debug install"],{}] INFO: Analysed target [["@","tutorial","","helloworld-debug install"],{}] INFO: Export targets found: 0 cached, 1 uncached, 0 not eligible for caching INFO: Discovered 7 actions, 0 tree overlays, 3 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld-debug install"],{}]. INFO: Processed 7 actions, 0 cache hits. INFO: Artifacts can be found in: /tmp/tutorial/.ext/debug/bin/helloworld [5f0ce4ed97000af42902c41f9d3b7d51343534a6:1570896:x] /tmp/tutorial/.ext/debug/include/fmt [3a5cb60e63f7480b150fdef0883d7a76e8a57a00:464:t] /tmp/tutorial/.ext/debug/include/greet/greet.hpp [63815ae1b5a36ab29efa535141fee67f3b7769de:53:f] /tmp/tutorial/.ext/debug/work/fmt [3a5cb60e63f7480b150fdef0883d7a76e8a57a00:464:t] /tmp/tutorial/.ext/debug/work/format.cc [ecb8cc79a6e9ff277db43876a11eccde40814ece:5697:f] /tmp/tutorial/.ext/debug/work/greet/greet.cpp [8c56239aabd4e23b9d170333d03f222e6938dcef:115:f] /tmp/tutorial/.ext/debug/work/greet/greet.hpp [63815ae1b5a36ab29efa535141fee67f3b7769de:53:f] /tmp/tutorial/.ext/debug/work/main.cpp [92f9b55228774b3d3066652253499395d9ebef31:76:f] /tmp/tutorial/.ext/debug/work/os.cc [04b4dc506005d38250803cfccbd9fd3b6ab30599:10897:f] INFO: Backing up artifacts of 1 export targets ``` To now debug the *helloworld* executable, we simply need to switch to the specified directory and run `gdb(1)`. ``` sh $ cd .ext/debug $ gdb bin/helloworld ``` This works out-of-the-box specifically because our tool keeps track of the logical staging of the artifacts (from direct and transitive dependencies) of targets, meaning it can easily mirror this staging in the install folder, thus ensuring a debugger finds all the symbols in the places it looks for by default. As such, this install directory can also be directly provided to an IDE (such as VSCode) as search location for debug symbols. Finally, note that not only the artifacts of the first-party library `greet` get staged, but also the artifacts of the third-party dependency `fmtlib`. This is due to the fact that the `"fmtlib"` export target is already configured (see its `TARGETS` file) to inherit the `"DEBUG"` flag from the environment, meaning that the `true` value set by the `"helloworld-debug"` target gets honored. In general, we recommend that export targets always allow the `"DEBUG"` flag through, specifically to ensure consumers have access to and can build the target library also in debug mode. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/getting-started.md000066400000000000000000000333271516554100600256660ustar00rootroot00000000000000Getting Started =============== In order to use *justbuild*, first make sure that `just`, `just-mr`, and `just-import-git` are available in your `PATH`. Creating a new project ---------------------- *justbuild* needs to know the root of the project worked on. By default, it searches upwards from the current directory till it finds a marker. Currently, we support three different markers: the files `ROOT` and `WORKSPACE` or the directory `.git`. Lets create a new project by creating one of those markers: ``` sh $ touch ROOT ``` Creating a generic target ------------------------- By default, targets are described in `TARGETS` files. These files contain a `JSON` object with the target name as key and the target description as value. A target description is an object with at least a single mandatory field: `"type"`. This field specifies which rule (built-in or user-defined) to apply for this target. A simple target that only executes commands can be created using the built-in `"generic"` rule, which requires at least one command and one output file or directory. To create such a target, create the file `TARGETS` with the following content: ``` {.jsonc srcname="TARGETS"} { "greeter": { "type": "generic" , "cmds": ["echo -n 'Hello ' > out.txt", "cat name.txt >> out.txt"] , "outs": ["out.txt"] , "deps": ["name.txt"] } } ``` In this example, the `"greeter"` target will run two commands to produce the output file `out.txt`. The second command depends on the input file `name.txt` that we need to create as well: ``` sh $ echo World > name.txt ``` Building a generic target ------------------------- To build a target, we need to run `just` with the subcommand `build`: ``` sh $ just build greeter INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","greeter"],{}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: out.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] $ ``` The subcommand `build` just builds the artifact but does not stage it to any user-defined location on the file system. Instead it reports a description of the artifact consisting of `git` blob identifier, size, and type (in this case `f` for non-executable file). To also stage the produced artifact to the working directory, use the `install` subcommand and specify the output directory: ``` sh $ just install greeter -o . INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","greeter"],{}]. INFO: Processed 1 actions, 1 cache hits. INFO: Artifacts can be found in: /tmp/tutorial/out.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] $ cat out.txt Hello World $ ``` Note that the `install` subcommand initiates the build a second time, without executing any actions as all actions are being served from cache. The produced artifact is identical, which is indicated by the same hash/size/type. If one is only interested in one of the final artifacts, one can also request via the `-P` option that this particular artifact be written to standard output after the build; if the target produces only a single artifact, the flag `-p` can be used instead as well (without the need of specifying the artifact). As all messages are reported to standard error, this can be used for both, interactively reading a text file, as well as for piping the artifact to another program. ``` sh $ just build greeter -P out.txt INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","greeter"],{}]. INFO: Processed 1 actions, 1 cache hits. INFO: Artifacts built, logical paths are: out.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] Hello World $ just build -p INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","greeter"],{}]. INFO: Processed 1 actions, 1 cache hits. INFO: Artifacts built, logical paths are: out.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] Hello World $ ``` In the last example we used that, if no target is specified, the lexicographically first one is taken. Alternatively, we could also directly request the artifact `out.txt` from *justbuild*'s CAS (content-addressable storage) and print it on the command line via: ``` sh $ just install-cas [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] Hello World $ ``` The canonical way of requesting an object from the CAS is, as just shown, to specify the full triple of hash, size, and type, separated by colons and enclosed in square brackets. To simplify usage, the brackets can be omitted and the size and type fields have the default values `0` and `f`, respectively. While the default value for the size is wrong for all but one string, the hash still determines the content of the file and hence the local CAS is still able to retrieve the file. So the typical invocation would simply specify the hash. ``` sh $ just install-cas 557db03de997c86a4a028e1ebd3a1ceb225be238 Hello World $ ``` Targets versus Files: The Stage ------------------------------- When invoking the `build` command, we had to specify the target `greeter`, not the output file `out.txt`. While other build systems allow requests specifying an output file, for *justbuild* this would conflict with a fundamental design principle: staging; each target has its own logical output space, the "stage", where it can put its artifacts. We can, without any problem, add a second target also generating a file `out.txt`. ``` {.jsonc srcname="TARGETS"} ... , "upper": { "type": "generic" , "cmds": ["cat name.txt | tr a-z A-Z > out.txt"] , "outs": ["out.txt"] , "deps": ["name.txt"] } ... ``` As we only request targets, no conflicts arise. ``` sh $ just build upper -p INFO: Requested target is [["@","","","upper"],{}] INFO: Analysed target [["@","","","upper"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","upper"],{}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: out.txt [83cf24cdfb4891a36bee93421930dd220766299a:6:f] WORLD $ just build greeter -p INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","greeter"],{}]. INFO: Processed 1 actions, 1 cache hits. INFO: Artifacts built, logical paths are: out.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] Hello World $ ``` While one normally tries to design targets in such a way that they don't have conflicting files if they should be used together, it is up to the receiving target to decide what to do with those artifacts. A built-in rule allowing to rearrange artifacts is `"install"`; a detailed description of this rule can be found in the documentation. In the simple case of a target producing precisely one file, the argument `"files"` can be used to remap that file. Note that the mapping is from the desired location to the target name (representing the single artifact of that target) as such a mapping is necessarily conflict free. ``` {.jsonc srcname="TARGETS"} ... , "both": {"type": "install", "files": {"hello.txt": "greeter", "upper.txt": "upper"}} ... ``` ``` sh $ just build both INFO: Requested target is [["@","","","both"],{}] INFO: Analysed target [["@","","","both"],{}] INFO: Discovered 2 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","both"],{}]. INFO: Processed 2 actions, 2 cache hits. INFO: Artifacts built, logical paths are: hello.txt [557db03de997c86a4a028e1ebd3a1ceb225be238:12:f] upper.txt [83cf24cdfb4891a36bee93421930dd220766299a:6:f] $ ``` Inspecting internals: Analyse ----------------------------- For our simple targets, it is easy to remember what they look like, and what the build steps are. For more complex projects this might no longer be the case. Therefore, *justbuild* tries to be very transparent about its understanding of the build. The command to do so is called `analyse` and also takes a target as argument. It shows all(!) the information available to a target depending on it. In the case of `greeter` this is just a single artifact. ``` sh $ just analyse greeter INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Result of target [["@","","","greeter"],{}]: { "artifacts": { "out.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"} }, "provides": { }, "runfiles": { } } INFO: Target tainted ["test"]. ``` As we can see, the artifact will be installed at the logical path `out.txt`. The artifact itself is generated by a build action, more precisely, it is the output `out.txt` of that action. Actions, in general, can have more than one output. For the target `both` we find the same artifact at a different output location in this case `hello.txt`. ``` sh $ just analyse both INFO: Requested target is [["@","","","both"],{}] INFO: Analysed target [["@","","","both"],{}] INFO: Result of target [["@","","","both"],{}]: { "artifacts": { "hello.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"}, "upper.txt": {"data":{"id":"f122c2af488a56c94d08650a9d799dbb910484409116b8e0c071e198082256ef","path":"out.txt"},"type":"ACTION"} }, "provides": { }, "runfiles": { "hello.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"}, "upper.txt": {"data":{"id":"f122c2af488a56c94d08650a9d799dbb910484409116b8e0c071e198082256ef","path":"out.txt"},"type":"ACTION"} } } INFO: Target tainted ["test"]. ``` We also see another technical detail. As an `install` target, `both` also has the same artifact in the `runfiles` arrangement of files. Having two dedicated arrangements of artifacts, that are both installed when the target is installed, can be useful to distinguish between the main artifact and files needed when building against that target (e.g., to distinguish a library from its header files). As a very generic built-in rule, however, `install` simply makes all files available in both arrangements. The canonical way to obtain all action definitions is to dump the action graph using the `--dump-graph` option. ``` sh $ just analyse both --dump-graph all-actions.json $ cat all-actions.json ``` As we know that the action was generated by the `greeter` target, was can also ask that target for the actions associated with it using the `--dump-actions` option. The `install` target does not generate any actions. ``` sh $ just analyse greeter --dump-actions - INFO: Requested target is [["@","","","greeter"],{}] INFO: Analysed target [["@","","","greeter"],{}] INFO: Result of target [["@","","","greeter"],{}]: { "artifacts": { "out.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"} }, "provides": { }, "runfiles": { } } INFO: Actions for target [["@","","","greeter"],{}]: [ { "command": ["sh","-c","echo -n 'Hello ' > out.txt\ncat name.txt >> out.txt\n"], "input": { "name.txt": { "data": { "path": "name.txt", "repository": "" }, "type": "LOCAL" } }, "output": ["out.txt"] } ] $ just analyse both --dump-actions - INFO: Requested target is [["@","","","both"],{}] INFO: Analysed target [["@","","","both"],{}] INFO: Result of target [["@","","","both"],{}]: { "artifacts": { "hello.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"}, "upper.txt": {"data":{"id":"f122c2af488a56c94d08650a9d799dbb910484409116b8e0c071e198082256ef","path":"out.txt"},"type":"ACTION"} }, "provides": { }, "runfiles": { "hello.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"}, "upper.txt": {"data":{"id":"f122c2af488a56c94d08650a9d799dbb910484409116b8e0c071e198082256ef","path":"out.txt"},"type":"ACTION"} } } INFO: Actions for target [["@","","","both"],{}]: [ ] $ ``` However, those actions are not owned by a target in any way. Actions are identified by their definition in the same way as `git` identifies a blob by its sequence of bytes and names it by a suitable hash value. So, let's add another target defining an action in the same way. ``` {.jsonc srcname="TARGETS"} ... , "another greeter": { "type": "generic" , "cmds": ["echo -n 'Hello ' > out.txt", "cat name.txt >> out.txt"] , "outs": ["out.txt"] , "deps": ["name.txt"] } ... ``` Then we find that it defines the same artifact. ``` sh $ just analyse 'another greeter' INFO: Requested target is [["@","","","another greeter"],{}] INFO: Analysed target [["@","","","another greeter"],{}] INFO: Result of target [["@","","","another greeter"],{}]: { "artifacts": { "out.txt": {"data":{"id":"28c9f5feb17361a5f755b19961b4c80c651586269786d93e58bfff5b0038c620","path":"out.txt"},"type":"ACTION"} }, "provides": { }, "runfiles": { } } ``` This is also the notion of equality used to detect conflicts when analysing targets before running any actions. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/hello-world.md000066400000000000000000000422451516554100600250100ustar00rootroot00000000000000Building C++ Hello World ======================== *justbuild* is a true language-agnostic (there are no more-equal languages) and multi-repository build system. As a consequence, high-level concepts (e.g., C++ binaries, C++ libraries, etc.) are not hardcoded built-ins of the tool, but rather provided via a set of rules. These rules can be specified as a true dependency to your project like any other external repository your project might depend on. Setting up the Multi-Repository Configuration --------------------------------------------- To build a project with multi-repository dependencies, we first need to provide a configuration that declares the required repositories. Before we begin, we need to declare where the root of our workspace is located by creating an empty file `ROOT`: ``` sh $ touch ROOT ``` Second, we also need to create the multi-repository configuration `repos.json` in the workspace root: ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc"} } } } ``` In that configuration, two repositories are defined: 1. The `"rules-cc"` repository located in the subdirectory `rules` of [just-buildsystem/rules-cc:7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6](https://github.com/just-buildsystem/rules-cc/tree/7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6), which contains the high-level concepts for building C/C++ binaries and libraries. 2. The `"tutorial"` repository located at `.`, which contains the targets that we want to build. It has a single dependency, which is the *rules* that are needed to build the target. These rules are bound via the open name `"rules"` to the just created repository `"rules-cc"`. In this way, the entities provided by `"rules-cc"` can be accessed from within the `"tutorial"` repository via the fully-qualified name `["@", "rules", "", ""]`; fully-qualified names (for rules, targets to build (like libraries, binaries), etc) are given by a repository name, a path specifying a directory within that repository (the "module") where the specification file is located, and a symbolic name (i.e., an arbitrary string that is used as key in the specification). The final repository configuration contains a single `JSON` object with the key `"repositories"` referring to an object of repository names as keys and repository descriptions as values. For convenience, the main repository to pick is set to `"tutorial"`. Description of the helloworld target ------------------------------------ For this tutorial, we want to create a target `helloworld` that produces a binary from the C++ source `main.cpp`. To define such a target, create a `TARGETS` file with the following content: ``` {.jsonc srcname="TARGETS"} { "helloworld": { "type": ["@", "rules", "CC", "binary"] , "name": ["helloworld"] , "srcs": ["main.cpp"] } } ``` The `"type"` field refers to the rule `"binary"` from the module `"CC"` of the `"rules"` repository. This rule additionally requires the string field `"name"`, which specifies the name of the binary to produce; as the generic interface of rules is to have fields either take a list of strings or a list of targets, we have to specify the name as a list (this rule will simply concatenate all strings given in this field). Furthermore, at least one input to the binary is required, which can be specified via the target fields `"srcs"` or `"deps"`. In our case, the former is used, which contains our single source file. Source files are also targets, but, as seen in the "Getting Started" section, not the only ones; instead of naming a source file, we could also have specified a `"generic"` target generating one (or many) of the sources of our binary. Now, the last file that is missing is the actual source file `main.cpp`: ``` {.cpp srcname="main.cpp"} #include int main() { std::cout << "Hello world!\n"; return 0; } ``` Building the helloworld target ------------------------------ To build the `helloworld` target, we need specify it on the `just-mr` command line: ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [bd36255e856ddb72c844c2010a785ab70ee75d56:17088:x] $ ``` Note that the target is taken from the `tutorial` repository, as it specified as the main repository in `repos.json`. If targets from other repositories should be build, the repository to use must be specified via the `--main` option. `just-mr` reads the repository configuration, fetches externals (if any), generates the actual build configuration, and stores it in its cache directory (by default under `$HOME/.cache/just`). Afterwards, the generated configuration is used to call the `just` binary, which performs the actual build. Note that these two programs, `just-mr` and `just`, can also be run individually. To do so, first run `just-mr` with `setup` and capture the path to the generated build configuration from stdout (above omitted from the log message as `"..."`) by assigning it to a shell variable (e.g., `CONF`). Afterwards, `just` can be called to perform the actual build by explicitly specifying the configuration file via `-C`, e.g.: ``` sh $ CONF=$(just-mr setup tutorial) $ just build -C $CONF helloworld ``` Note that `just-mr` only needs to be run the very first time and only once again whenever the `repos.json` file is modified. By default, the BSD-default compiler front-ends (which are also defined for most Linux distributions) `cc` and `c++` are used for C and C++ (variables `"CC"` and `"CXX"`). If you want to temporarily use different defaults, you can use `-D` to provide a JSON object that sets different default variables. For instance, to use Clang as C++ compiler for a single build invocation, you can use the following command to provide an object that sets `"CXX"` to `"clang++"`: ``` sh $ just-mr build helloworld -D'{"CXX":"clang++"}' INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld","-D{\"CXX\":\"clang++\"}"] INFO: Requested target is [["@","tutorial","","helloworld"],{"CXX":"clang++"}] INFO: Analysed target [["@","tutorial","","helloworld"],{"CXX":"clang++"}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{"CXX":"clang++"}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [a1e0dc77ec6f171e118a3e6992859f68617a2c6f:16944:x] $ ``` Defining project defaults ------------------------- To define a custom set of defaults (toolchain and compile flags) for your project, you need to create a separate file root for providing required `TARGETS` file, which contains the `"defaults"` target that should be used by the rules. This file root is then used as the *target root* for the rules, i.e., the search path for `TARGETS` files. In this way, the description of the `"defaults"` target, while logically part of the rules repository is physically located in a separate directory to keep the rules repository independent of these project-specific definitions. We will call the new file root `tutorial-defaults` and need to create a module directory `CC` in it: ``` sh $ mkdir -p ./tutorial-defaults/CC ``` In that module, we need to create the file `tutorial-defaults/CC/TARGETS` that contains the target `"defaults"` and specifies which toolchain and compile flags to use; it has to specify the complete toolchain, but can specify a `"base"` toolchain to inherit from. In our case, we don't use any base. ``` {.jsonc srcname="tutorial-defaults/CC/TARGETS"} { "defaults": { "type": ["CC", "defaults"] , "CC": ["cc"] , "CXX": ["c++"] , "ADD_COMPILE_FLAGS": ["-O2", "-Wall"] , "AR": ["ar"] , "DWP": ["dwp"] , "PATH": ["/bin", "/usr/bin"] } } ``` Here we used `"ADD_COMPILE_FLAGS"` to add flags for both, `C` and `C++` compilation. Those flags are added to the ones inherited from `"base"`, in our case (as we did not specify a `"base"`) the empty list. There are also `"ADD_CFLAGS"` and `"ADD_CXXFLAGS"` if we want to add flags for just `C` or just `C++`. Finally, there is also the possibility to explicitly specify `"CFLAGS"` and `"CXXFLAGS"` (completely ignoring anything inherited). To use the project defaults, modify the existing `repos.json` to reflect the following content: ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc"} } , "tutorial-defaults": { "repository": {"type": "file", "path": "./tutorial-defaults"} } } } ``` Note that the `"defaults"` target uses the rule `["CC", "defaults"]` without specifying any external repository (e.g., `["@", "rules", ...]`). This is because `"tutorial-defaults"` is not a full-fledged repository but merely a file root that is considered local to the `"rules-cc"` repository. In fact, the `"rules-cc"` repository cannot refer to any external repository as it does not have any defined bindings. The naming for rules follows the same scheme we've already seen for targets, so a single string refers to an entity in the same directory. As our `"defaults"` target is in the directory `"CC"` of the rules repository we could also have written the rule `"type"` simply as `"defaults"`. To rebuild the project, we rerun `just-mr` (note that due to configuration changes, we expect the intermediary configuration file hash reported to also change): ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [0d5754a83c7c787b1c4dd717c8588ecef203fb72:16992:x] $ ``` Note that the output binary has changed due to different defaults. In this tutorial we simply set the correct parameters of the defaults target. It is, however, not necessary to remember all the fields of a rule; we can always ask `just` to present us the available field names and configuration variables together with any documentation the rule author provided. For this, we use the `describe` subcommand; as we're interested in a target of the `rules-cc` repository, which is not the default repository, we also have to specify the repository name. ``` sh $ just-mr --main rules-cc describe CC defaults ``` Of course, the `describe` subcommand works generically on all targets. For example, by asking to describe our `helloworld` target, we will get reminded about all the various fields and relevant configuration variables of a C++ binary. ``` sh $ just-mr describe helloworld ``` Modeling target dependencies ---------------------------- For demonstration purposes, we will separate the print statements into a static library `greet`, which will become a dependency to our binary. Therefore, we create a new subdirectory `greet` ``` sh $ mkdir -p ./greet ``` with the files `greet/greet.hpp`: ``` {.cpp srcname="greet/greet.hpp"} #include void greet(std::string const& s); ``` and `greet/greet.cpp`: ``` {.cpp srcname="greet/greet.cpp"} #include "greet.hpp" #include void greet(std::string const& s) { std::cout << "Hello " << s << "!\n"; } ``` These files can now be used to create a static library `libgreet.a`. To do so, we need to create the following target description in `greet/TARGETS`: ``` {.jsonc srcname="greet/TARGETS"} { "greet": { "type": ["@", "rules", "CC", "library"] , "name": ["greet"] , "hdrs": ["greet.hpp"] , "srcs": ["greet.cpp"] , "stage": ["greet"] } } ``` Similar to `"binary"`, we have to provide a name and source file. Additionally, a library has public headers defined via `"hdrs"` and an optional staging directory `"stage"` (default value `"."`). The staging directory specifies where the consumer of this library can expect to find the library's artifacts. Note that this does not need to reflect the location on the file system (i.e., a full-qualified path like `["com", "example", "utils", "greet"]` could be used to distinguish it from greeting libraries of other projects). The staging directory does not only affect the main artifact `libgreet.a` but also it's *runfiles*, a second set of artifacts, usually those a consumer needs to make proper use the actual artifact; in the case of a library, the runfiles are its public headers. Hence, the public header will be staged to `"greet/greet.hpp"`. With that knowledge, we can now perform the necessary modifications to `main.cpp`: ``` {.cpp srcname="main.cpp"} #include "greet/greet.hpp" int main() { greet("Universe"); return 0; } ``` The target `"helloworld"` will have a direct dependency to the target `"greet"` of the module `"greet"` in the top-level `TARGETS` file: ``` {.jsonc srcname="TARGETS"} { "helloworld": { "type": ["@", "rules", "CC", "binary"] , "name": ["helloworld"] , "srcs": ["main.cpp"] , "private-deps": [["greet", "greet"]] } } ``` Note that there is no need to explicitly specify `"greet"`'s public headers here as the appropriate artifacts of dependencies are automatically added to the inputs of compile and link actions. The new binary can be built with the same command as before (no need to rerun `just-mr`): ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [a0e593e4d52e8b3e14863b3cf1f80809143829ca:17664:x] $ ``` To only build the static library target `"greet"` from module `"greet"`, run the following command: ``` sh $ just-mr build greet greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","greet","greet"] INFO: Requested target is [["@","tutorial","greet","greet"],{}] INFO: Analysed target [["@","tutorial","greet","greet"],{}] INFO: Discovered 2 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","tutorial","greet","greet"],{}]. INFO: Processed 2 actions, 2 cache hits. INFO: Artifacts built, logical paths are: greet/libgreet.a [83ed406e21f285337b0c9bd5011f56f656bba683:2992:f] (1 runfiles omitted.) $ ``` The omitted (i.e., not shown but still built) runfile is the header file. As mentioned in the introduction to `just analyse` this is a typical use of that second artifact arrangement. We can also have a look at the other information that library provides. ``` sh $ just-mr analyse greet greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","greet","greet"] INFO: Requested target is [["@","tutorial","greet","greet"],{}] INFO: Analysed target [["@","tutorial","greet","greet"],{}] INFO: Result of target [["@","tutorial","greet","greet"],{}]: { "artifacts": { "greet/libgreet.a": {"data":{"id":"d964a2747015935adc5fd7f06bbd910d5dde99e990436be0b1f7034270b5b11d","path":"work/greet/libgreet.a"},"type":"ACTION"} }, "provides": { "compile-args": [ ], "compile-deps": { }, "debug-hdrs": { }, "debug-srcs": { }, "dwarf-pkg": { }, "link-args": [ "greet/libgreet.a" ], "link-deps": { }, "lint": [ ], "package": { "cflags-files": {}, "ldflags-files": {}, "name": "greet" }, "run-libs": { }, "run-libs-args": [ ] }, "runfiles": { "greet/greet.hpp": {"data":{"path":"greet/greet.hpp","repository":"tutorial"},"type":"LOCAL"} } } $ ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/invocation-logging.md000066400000000000000000000112741516554100600263530ustar00rootroot00000000000000# Invocation Logging and Profiling For large projects, it can be helpful to find out which actions make, e.g., the runs of the continuous integration slow. `just` has an option `--profile` that instructs it to write a profile file to the specified location on disk. That profile file contains (among other things, see `just-profile(5)`) for each action that was considered during the build whether it was cached, and, if not, its run time. These actions can be associated to targets via the `"origins"` filed in the action graph that can be obtained via the `--dump-graph` option. However, not only the run time of actions in a particular build matters, but also how often an action has to be rerun because some input changed. Therefore, we need to collect profiling data over many invocations. To do so, `just-mr` honors an option `"invocation log"` in its rc file. For that key, several subkeys can be specified with the most important being `"directory"`; the location object defined there specifies the directory where the logging happens (always on the local machine, even if remote-execution is used). Inside this directory, a subdirectory is used for each project; the project can be specified by the subkey `"project id"`. Typically, setting the project identifier is delegated to an rc file in the workspace. So, if we have `~/.just-mrrc` with contents ``` json { "rc files": [{"root": "workspace", "path": "etc/rc.json"}] , "invocation log": {"directory": {"root": "home", "path": "log/justbuild"}} } ``` and the project contains an `etc/rc.json` with ``` json { "invocation log": {"project id": "hot-new-product"}} ``` then for each build triggered through `just-mr` a new subdirectory of `~/log/justbuild/hot-new-product` will be created. The name of that subdirectory consists of date and time, followed by a universally-unique identifier. In particular, parallel invocations are no problem, even if invocation logging is activated. The prefixing with date and time allows simple aggregation or cleanup over fixed periods of time (like monthly, daily, hourly, etc). Of course, just creating an empty directory for each invocation of `just-mr` is not very useful. Therefore, more subkeys can be specified. - `"--profile"` specifies the file name within the invocation-log directory to be used when generating the `--profile` option in the command line for the `just` invocation. - `"--dump-graph"` does the same for the `--dump-graph` option. - `"--dump-artifacts-to-build"` does the same for the `--dump-artifacts-to-build` option. - `"--dump-artifacts"` does the same for the `--dump-artifacts` option; while not directly useful for profiling, browsing the final artifacts (including the test logs) can be useful to understand a particular invocation. - `"metadata"` specifies a file name inside the invocation directory a metadata file should be written by `just-mr`; that file contains, in particular, the full command line that is executed and the blob identifier of the repository configuration file used in that invocation of `just` (if any). - `"context variables"` specifies a list of environment variables for which the value should be recorded in the metadata file; while `just` is designed to deliberately ignore environment variables for the build, environment variables can be used to communicate some context for the invocation, especially when run on a CI system. This can later also be used for an analysis based on a more fine-grained sharding. So, if invocation logging is desired, the relevant part of a typical `~/.just-mrrc` file could look as follows. ``` json { "rc files": [{"root": "workspace", "path": "etc/rc.json"}] , "invocation log": { "directory": {"root": "home", "path": "log/justbuild"} , "metadata": "meta.json" , "context variables": ["CI_MERGE_REQUEST_IID", "CI_COMMIT_SHA"] , "--profile": "profile.json" , "--dump-graph": "graph.json" , "--dump-artifacts-to-build": "to-build.json" , "--dump-artifacts": "artifacts.json" } } ``` If some shared infrastructure (like a network file system) is available, it usually is a good idea to choose for `"directory"` a `"system"` path rather than a `"home"` one. In any case, it is advisable to set up some form of cronjob to rotate the invocation logs, as they can get quite large. Of course, the main motivation for invocation logging is doing statistical analysis later. However, it can also be useful to browse through the most recent invocations, looking, e.g., at failed actions and their output, or actions that took particularly long. In the `justbuild` source tree, under `doc/invocations-http-server` there is a simple application serving one directory of invocation logs, i.e., the logs for one particular project id. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/just-execute.org000066400000000000000000000330731516554100600253730ustar00rootroot00000000000000* Single-node remote execution service: ~just execute~ ~just execute~ starts a single-node remote build execution service in the environment in which the command has been issued. Having the possibility to easily create a remote build execution service can improve the developing experience where the build environment (and the cache) can/should be shared among the developers. For example (and certainly not limited to) - when developers build on the same machine. It will allow multiple users to build in the same environment and share the cache, thus avoiding duplicated work. - quickly set up a testing environment that can be used by other developers. For the sake of completeness, these are the files used to compile the examples #+BEGIN_SRC latex-hello-world/ +--hello.tex +--repos.json +--TARGETS #+END_SRC They read as follows File ~repos.json~: #+SRCNAME: repos.json #+BEGIN_SRC js { "main": "tutorial" , "repositories": { "latex-rules": { "repository": { "type": "git" , "branch": "master" , "commit": "ffa07d6f3b536f1a4b111c3bf5850484bb9bf3dc" , "repository": "https://github.com/just-buildsystem/rules-typesetting" } } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"latex-rules": "latex-rules"} } } } #+END_SRC File ~TARGETS~: #+SRCNAME: TARGETS #+BEGIN_SRC js { "tutorial": { "type": ["@", "latex-rules", "latex", "latexmk"] , "main": ["hello"] , "srcs": ["hello.tex"] } } #+END_SRC File ~hello.tex~: #+SRCNAME: hello.tex #+BEGIN_SRC tex \documentclass[a4paper]{article} \author{JustBuild developers} \date{} \title {just execute} \begin{document} \maketitle Hello from \LaTeX! \end{document} #+END_SRC ** Simple usage of ~just execute~ in the same environment In this first example, we simply call ~just execute~ and the environment of the caller is made available. We therefore recommend to have a dedicated non-privileged ~build~ user to run the execution service. In the following, we will use ~%~ to indicate the prompt of the ~build~ user, ~$~ for a /normal/ user. To enable such a single-node execution service, it is sufficient to type on one shell (as ~build~ user) #+BEGIN_SRC bash % just execute -p #+END_SRC Where ~~ is a port number which is supposed to be available. By default, the native ~git~-based protocol will be used, but it is also possible to use the original protocol with ~sha256~ hashes by providing the ~--compatible~ option. #+BEGIN_SRC bash % just execute --compatible -p #+END_SRC This is particularly useful when providing the remote-execution service to a different build tool. To use it, as a /normal/ user, on a different shell type #+BEGIN_SRC bash $ just [...] -r localhost: #+END_SRC Let's run these commands to understand the output. #+BEGIN_SRC bash % just execute -p 8080 INFO: execution service started: {"interface":"127.0.0.1","pid":4911,"port":8080} #+END_SRC Once the execution service is started, it logs out three essential data: - which interface is used (in this case, the default one, which is the loopback device) - the pid number (number will always change) - the used port To exploit the execution service, run from a different shell #+BEGIN_SRC bash $ just [...] -r localhost:8080 #+END_SRC *** Use a random port If we don't need (or know) a fixed port number, we can simply omit the ~-p~ option. In this case, ~just execute~ will listen to a random free port. #+BEGIN_SRC bash % just execute INFO: execution service started: {"interface":"127.0.0.1","pid":7217,"port":33841} #+END_SRC The port number can be different each time we invoke the above command. Finally, to connect to the remote endpoint, type #+BEGIN_SRC bash $ just [...] -r localhost:33841 #+END_SRC *** Info file Copying and pasting port numbers and pids can be error-prone/unfeasible if we manage several/many execution service instances. Therefore, the invocation of ~just execute~ can be decorated with the option ~--info-file ~, which will store, in JSON format, in ~~ the interface, pid, and port bound to the running instance. The user can then easily parse this file to extract the required information. For example #+BEGIN_SRC bash % just execute --info-file /tmp/foo.json INFO: execution service started: {"interface":"127.0.0.1","pid":7680,"port":44115} #+END_SRC #+BEGIN_SRC bash $ cat /tmp/foo.json {"interface":"127.0.0.1","pid":7680,"port":44115} #+END_SRC Please note that the info file will /not be automatically deleted/ when the user terminates the service. The user is responsible for eventually removing it from the file system. *** Enable mTLS It is worth mentioning that mTLS must be enabled when the execution service starts, and it cannot be activated (or deactivated) while the instance runs. #+BEGIN_SRC bash % just execute [...] --tls-ca-cert --tls-server-cert --tls-server-key #+END_SRC When a client connects, it must pass the same ~CA certificate~ and its pair of certificate and private key, which the used certified authority has signed. #+BEGIN_SRC bash $ just [...] --tls-ca-cert --tls-client-cert --tls-client-key #+END_SRC **** How to generate self-signed certificates This section does not pretend to be an exhaustive guide to the generation and management of certificates, which is well beyond the aim of this tutorial. We just want to provide a minimal reference for let users start using mTLS and having the benefits of mutual authentication. ***** Certification Authority certificate As a first step, we need a Certification Authority certificate (~ca.crt~) #+BEGIN_SRC bash % openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:4096 -keyout ca.key -out ca.crt #+END_SRC ***** Server certificate and key If the clients will connect using the loopback device, i.e., the users are logged in the same machine where ~just execute~ will run, the /server certificates/ can be generate with the following instructions #+BEGIN_SRC bash % openssl req -new -nodes -newkey rsa:4096 -keyout server.key -out server.csr -subj "/CN=localhost" % openssl x509 -req -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 0 -out server.crt % rm server.csr #+END_SRC bash On the other hand, if the clients will connect from a different machine, and ~just execute~ will use a different interface (see [[Expose a particular interface]] below), the steps are a bit more involved. We need an additional configuration file where we state the ip address of the used interface. For example, if the interface ip address is ~192.168.1.14~, we will write #+BEGIN_SRC bash % cat << EOF > ssl-ext-x509.cnf [v3_ca] subjectAltName = IP.1:192.168.1.14 EOF #+END_SRC Then, the pair of certificate and pair can be obtained with #+BEGIN_SRC bash % openssl req -new -nodes -newkey rsa:4096 -keyout server.key -out server.csr -subj "/CN=localhost" % openssl x509 -req -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 0 -out server.crt -extensions v3_ca -extfile ssl-ext-x509.cnf % rm server.csr #+END_SRC bash ***** Client certificate and key The client, which needs the ~ca.crt~ and ~ca.key~ files, can run the following #+BEGIN_SRC bash $ openssl req -new -nodes -newkey rsa:4096 -keyout client.key -out client.csr $ openssl x509 -req -days 365 -signkey client.key -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out client.crt $ rm client.csr #+END_SRC *** Expose a particular interface To use an interface different from the loopback one, we have to list it with the ~-i~ option #+BEGIN_SRC bash $ just execute -i 192.168.1.14 -p 8080 --tls-ca-cert --tls-server-cert --tls-server-key INFO: execution service started: {"interface":"192.168.1.14","pid":7917,"port":8080} #+END_SRC If the interface is accessible from another machine, it is also recommended to enable mutual TLS (mTLS) authentication. ** Managing multiple build environments Since multiple instances of ~just execute~ can run in parallel (listening at different ports), the same machine can be the worker for various projects. However, to avoid conflicts between the dependencies and to guarantee a clean environment for each project, it is recommended that ~just execute~ is invoked from within a container or a chroot environment. In the following sections, we will set up, step by step, a dedicated execution service for compiling latex documents in these two scenarios. *** How to run ~just execute~ inside a chroot environment **** TL;DR - create a suitable chroot environment - chroot into it - run ~just execute~ from there - in a different shell, ~just build -r :~ **** Full latex chroot: walkthrough This short tutorial will use ~debootstrap~ and ~schroot~ to create and enter the chroot environment. Of course, different strategies/programs can be used. ***** Prepare the root file system Install debian bullseye in directory ~/chroot/bullseye-latex~ #+BEGIN_SRC bash sudo debootstrap bullseye /chroot/bullseye-latex #+END_SRC ***** Create a configuration file ~schroot~ needs a proper configuration file, which can be generated as follows #+BEGIN_SRC bash $ echo "[bullseye-latex] description=bullseye latex env directory=/chroot/bullseye-latex root-users=$(whoami) users=$(whoami) type=directory" | sudo tee /etc/schroot/chroot.d/bullseye-latex #+END_SRC Note that ~type=directory~, apart from performing the necessary bindings, will make ~$HOME~ shared between the host and chroot environment. While this can be useful for sharing artifacts, the user should specify a ~--local-build-root~ (aka, the cache root) different from the default one to avoid conflicts between the host and the chroot environment. ***** Install required packages in the chroot environment ~schroot~ also allows running commands inside the environment by stating it after the ~--~ #+BEGIN_SRC bash $ schroot -c bullseye-latex -u root -- sh -c 'apt update && apt install -y texlive-full' #+END_SRC ***** Start the execution service To start the execution service inside the chroot environment run #+BEGIN_SRC bash $ schroot -c bullseye-latex -- /bin/just execute --local-build-root ~/.cache/chroot/bullseye-latex -p 8080 #+END_SRC We assumed that the binary ~just~ is available in the chroot environment at the path ~/bin/just~. If you don't know how to make ~just~ available in the chroot environment, read the section [[How to have the binary just inside the chroot environment]] below. Since the ~$HOME~ is shared, specifying a local build root (aka, cache root) different from the default is highly recommended. For convenience, we also set a port (using the flag ~-p~) that the execution service will listen to. If the chosen port is available, the following output should be produced (note that the pid number might be different). #+BEGIN_SRC bash INFO: execution service started: {"interface":"127.0.0.1","pid":48880,"port":8080} #+END_SRC For example, let's compile the example listed in the introduction #+BEGIN_SRC bash $ just-mr -C repos.json install -o . -r localhost:8080 #+END_SRC which should report #+BEGIN_SRC bash INFO: Performing repositories setup INFO: Found 2 repositories to set up INFO: Setup finished, exec ["just","install","-C","...","-o",".","-r","localhost:8080"] INFO: Requested target is [["@","tutorial","doc/just-execute/latex-hello-world","tutorial"],{}] INFO: Analysed target [["@","tutorial","doc/just-execute/latex-hello-world","tutorial"],{}] INFO: Discovered 1 actions, 0 trees, 1 blobs INFO: Building [["@","tutorial","doc/just-execute/latex-hello-world","tutorial"],{}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts can be found in: /tmp/work/doc/just-execute/latex-hello-world/hello.pdf [25e05d3560e344b0180097f21a8074ecb0d9f343:37614:f] #+END_SRC In the shell where ~just execute~ is running, this line should have appeared, witnessing that the compilation happened on the remote side #+BEGIN_SRC bash INFO (execution-service): Execute 6237d87faed1ec239512ad952eeb412cdfab372562 #+END_SRC *** How to start ~just execute~ inside a docker container Building inside a container is another strategy to ensure no undeclared dependencies are pulled and to build in a fixed environment. We will replicate what we did for the chroot environment and create a suitable docker image. **** Build a suitable docker image Let's write a ~Dockerfile~ that has ~just execute~ as ~ENTRYPOINT~. We assume the binary ~just~ is available inside the container at path ~/bin/just~. The easiest way is to use a [[https://github.com/just-buildsystem/justbuild-static-binaries][static just binary]] and copy it into the container. #+SRCNAME: Dockerfile #+BEGIN_SRC docker FROM debian:bullseye-slim COPY ./just /bin/just RUN apt update RUN apt install -y --no-install-recommends texlive-full ENTRYPOINT ["/bin/just", "execute"] #+END_SRC We build the image with #+BEGIN_SRC bash $ sudo docker image build -t bullseye-latex . #+END_SRC Finally, we can start the execution service #+BEGIN_SRC bash $ docker run --network host --name execute-latex -p 8080 #+END_SRC From a different shell, we can build the latex hello world example listed in the introduction running #+BEGIN_SRC bash $ just-mr -C repos.json install -o . -r localhost:8080 #+END_SRC Note that the cache that ~just execute~ populates is confined within the container. The cache is gone if the container is restarted (or the pc rebooted). If you want the cache to survive the container life cycle, you can bind a "host directory" within the container as follows #+BEGIN_SRC bash $ docker run --network host --name execute-latex --mount type=bind,source="${HOME}/.cache",target=/cache bullseye-latex -p 8080 --local-build-root /cache/docker/latex #+END_SRC just-buildsystem-justbuild-b1fb5fa/doc/tutorial/just-lock.md000066400000000000000000000315011516554100600244640ustar00rootroot00000000000000Multi-repository configuration management: `just-lock` ====================================================== The multi-repository build configuration used as input by `just-mr` acts for all intents and purposes as a _lockfile_ for *justbuild* projects, containing the pinned versions of all content-fixed repositories. This is expected to be stored and shipped together with the source code, allowing a consistent development environment for all users. With dependencies of projects coming in many shapes and forms, `just-lock` is a tool for generating and maintaining the multi-repositories configuration (lock)file of *justbuild* projects. This tool performs several functionalities, such as importing dependencies with repository composition (extending the functionality available in the `just-import-git` tool), automatic deduplication of identical transitive dependencies (a functionality available also standalone in the `just-deduplicate-repos` tool), or setup of repository clones for local development. For the purposes of this tutorial, we will focus only on the dependency import aspects. Basic use of `just-lock` ------------------------ In order to produce the multi-repository build configuration file, `just-lock` expects an input configuration file. The format of this file is an extension of the one of `just-mr` and it is read as a `JSON` object. The file defines four elements: - the description of local repositories, as the value for mandatory field `"repositories"`. This usually defines local checkouts, patches, overlays. - the description of remote dependencies, as the optional value for field `"imports"`. For *justbuild* dependencies, their multi-repository configuration (lock)file is taken as the ground truth for importing repository descriptions into the current project. Non-*justbuild* dependencies can also be described here, but can only be imported as-is, with overlays defining how such source code should be integrated (as it will be shown below). - the repository to consider as the main entry point for the build, as the value of the optional field `"main"`. - a list of repository names, as the value of the field `"keep"`. These names will be kept in the final configuration during the automatic process of deduplicating any repeated repositories brought in as transitive dependencies from different imports. For the purposes of this tutorial, the use of this field is not exemplified. ### The input file For the following, we return to the *hello_world* example from the section on [*Building Third-party dependencies*](./third-party-software.md), which depends on the open-source project [fmtlib](https://github.com/fmtlib/fmt), in the setup that enables high-level target caching. We define the following `repos.in.json` input configuration file for `just-lock`: ``` {.jsonc srcname="repos.in.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": "rules-cc-rules-sources" , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "fmt-targets-layer": { "repository": { "type": "file" , "path": "./fmt-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": "fmtlib-sources" , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } , "imports": [ { "source": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "url": "https://github.com/just-buildsystem/rules-cc.git" , "repos": [{"alias": "rules-cc-rules-sources", "repo": "rules"}] } , { "source": "git" , "branch": "8.1.1" , "url": "https://github.com/fmtlib/fmt.git" , "repos": [{"alias": "fmtlib-sources"}] , "as plain": true } ] } ``` As already mentioned, the `"imports"` field provides a description of the external (usually third-party) dependencies of a project. In our case, we use the `rules-cc` and `fmtlib` libraries, described as `git` sources. The first thing to note is the use of overlays. In our simple project, as we do not want to import anything other than the source trees of the needed dependencies, we create the `"rules-cc-rules-sources"` and `"fmtlib-sources"` names, which will bind the workspace roots of our `"rules-cc"` and `"fmtlib"` repositories, respectively, to the descriptions of the remote repositories providing the necessary source trees. The first import description object is for `rules-cc`, which is a *justbuild* project hosted as a Git repository, from which we would like to import its `"rules"` subdirectory. Thankfully, that project offers a useful shorthand by defining in its own locked repositories configuration file the `"rules"` overlay repository pointing to the respective subdirectory. `just-lock` will read that configuration file in order to produce the resulting configuration, caching any fetched source trees. It is thus highly recommended that the same build root as the one subsequently used by `just-mr` is provided to `just-lock` (via the `--local-build-root` option, with same default behaviour as in `just-mr`). In the case of `fmtib`, which is also hosted as a Git repository but does not provide a *justbuild* configuration file, we can only import it as-is, as a complete repository, signaled by setting the `"as plain"` flag. Do note that in this case we can limit ourselves to also just providing the `"branch"` field, and not also a specific commit. This is because `just-lock` automatically interrogates the remote in order to retrieve the top commit associated to a certain reference (in this case, a release tag) and pin it to a hard reference (in this case, the commit identifier) into the output configuration. This is a useful feature, as it is often the case that information about dependencies comes in the form of "loose" references, such as release tags or even simply the remote location of a distfile, but which `just-lock` then will pin down to a content-defined reference, such as a commit, blob, or tree identifier. ### Generating the configuration We generate an output configuration file `repos.out.json` by running `just-lock` with the appropriate arguments, then we build the `helloworld` target with this configuration: ``` sh $ just-lock -C repos.in.json -o repos.out.json [...] $ $ just-mr -C repos.out.json build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 1 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 4 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` As it can be seen, everything comes from cache and the new configuration behaves identical to the manually written one. If available, one can inspect the difference in content between the two files using the `jq` command-line tool: ``` sh $ diff -y <(jq --sort-keys . repos.out.json) <(jq --sort-keys . repos.json) { { "main": "tutorial", "main": "tutorial", "repositories": { "repositories": { "fmt-targets-layer": { "fmt-targets-layer": { "repository": { "repository": { "path": "./fmt-layer", "path": "./fmt-layer", "pragma": { "pragma": { "to_git": true "to_git": true }, }, "type": "file" "type": "file" } } }, }, "fmtlib": { "fmtlib": { "bindings": { "bindings": { "rules": "rules-cc" "rules": "rules-cc" }, }, "repository": "fmtlib-sources", < "target_root": "fmt-targets-layer" < }, < "fmtlib-sources": { < "repository": { "repository": { "branch": "8.1.1", "branch": "8.1.1", "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9", "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9", "repository": "https://github.com/fmtlib/fmt.git", "repository": "https://github.com/fmtlib/fmt.git", "type": "git" "type": "git" } | }, > "target_root": "fmt-targets-layer" }, }, "rules-cc": { "rules-cc": { "repository": "rules-cc-rules-sources", < "rule_root": "rules-cc", < "target_root": "tutorial-defaults" < }, < "rules-cc-rules-sources": { < "repository": { "repository": { "branch": "master", "branch": "master", "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6", "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6", "repository": "https://github.com/just-buildsystem/ru "repository": "https://github.com/just-buildsystem/ru "subdir": "rules", "subdir": "rules", "type": "git" "type": "git" } | }, > "rule_root": "rules-cc", > "target_root": "tutorial-defaults" }, }, "tutorial": { "tutorial": { "bindings": { "bindings": { "format": "fmtlib", "format": "fmtlib", "rules": "rules-cc" "rules": "rules-cc" }, }, "repository": { "repository": { "path": ".", "path": ".", "type": "file" "type": "file" } } }, }, "tutorial-defaults": { "tutorial-defaults": { "repository": { "repository": { "path": "./tutorial-defaults", "path": "./tutorial-defaults", "pragma": { "pragma": { "to_git": true "to_git": true }, }, "type": "file" "type": "file" } } } } } } } } ``` Except for the two overlays, kept from the `just-lock` input file configuration, the two configuration files have the same content. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/just-serve.md000066400000000000000000000770331516554100600246720ustar00rootroot00000000000000Dependency management: `just serve` =================================== `just serve` starts a remote target-level cache service (for ease called from now on the *serve service*) that can maintain and provide project dependencies via an associated remote-execution endpoint. The serve service is also responsible, in the case of a target-level cache miss when performing the target lookup, to build the relevant target (by dispatching it to a remote execution endpoint) and thus remember the result in its cache for subsequent requests. It has to be mentioned that we are only interested here in `export` targets in content-fixed repositories, as those are the ones eligible to be cached in the first place. The usefulness of such a service are several - projects can have many dependencies. These tend to not be part of the build environment and are instead built from source, but with *justbuild* these result in content-fixed repositories (including logically, via the `"to_git"` pragma), increasing reproducibility and easing target caching. - projects tend to share dependencies. By connecting to the same target-level cache service, and as dependencies tend to be updated less frequently than the code being actively developed, projects can expect frequent target cache hits, reducing overall build times. - easier maintenance of necessary archives, promoting reproducibility and improving auditing. - projects can avoid having to fetch their dependencies locally, thus reducing the amount of data needed to be present for a successful build. The way this service interacts with the other *justbuild* components is described in the following diagram: ┌───────────┐ ┌───────────────────>│ execution │ │ ┌─────────────────┤ endpoint │ │ │ └──────┬────┘ build │ │ artifacts ^ │ dispatch │ │ │ │ │ v │ │ ┌───┴───────┐ │ │ │ serve │ │ │ │ endpoint │ │ │ └───────────┘ │ │ ^ │ │ │ │ │ │ │ │ ...........│.......................│.│.......... │ │ │ │ │ │ basic │ build │ │ artifacts RPCs │ dispatch │ │ │ │ │ │ ┌────────┐ │ │ │ │ ├──────┘ │ └──────>│ client │ │ │ │<───────┘ └────────┘ As can be seen, the serve endpoint doubles as another client of the remote-execution endpoint, able to dispatch builds for export targets it has no cache entry for and retrieve the resulting build artifacts. In fact, any CAS object that should normally need to be transacted between the client and the serve endpoint happens only via the remote-execution endpoint, with direct communication being restricted to basic RPCs (which have minimal information and are thus very small). This reduces the communication taking place between the client and the serve endpoint, while ensuring that the remote CAS has all the opportunities to cache entries between builds. This comes at a slight cost of extra communication between the serve and execution endpoints (to sync various requested entries), however in a typical deployment these endpoints are expected to be close network-wise. In the remainder of this tutorial section, we will first cover some basics of setting up the serve service, then we will showcase how to set up and use `just serve` to provide first- and third-party dependencies to a *justbuild* project by expanding on a previously covered example scenario. Basic usage of `just serve` --------------------------- To simplify usage, `just serve` supports only one argument, the path to a configuration file containing all the options needed to set up the service. Therefore the serve service can simply be started by typing in a shell ```sh $ just serve ``` The format of the configuration file is `JSON`. For the purposes of this tutorial section, we will consider from now on as our configuration file the path `.just-servec` in the current working directory. Let us see what calling `just serve` with an empty configuration file does and interpret the result. Thus, using a configuration file with content ``` {.jsonc srcname=".just-servec"} {} ``` results in ``` sh $ just serve .just-servec INFO: serve and execute services started: {"interface":"127.0.0.1","pid":3728424,"port":34429} ``` First of all, the message mentions two services: `serve` and `execute`. As a reminder from the tutorial section on *Building Third-party Software*, running `just execute` starts a single-node remote build execution service in the current environment. Our `just serve` dependency management service always requires an associated remote-execution endpoint to be provided in order for various artifacts or Git objects to be transacted with a client via the CAS of the remote-execution endpoint. Therefore, calling `just serve` without specifying an existing remote-execution endpoint automatically creates for us a single-node remote-execution service, exposed at the same socket address. Once the serve service is started, it logs out three pieces of data: - which interface is used. In this case, the default one, which is the loopback device. - the pid number. This changes with each invocation. - the used port. In this case, the default one, a random free port was automatically chosen. Now the serve service can be used by running, in a different shell ``` sh $ just [...] -R localhost:34429 # or $ just-mr -R localhost:34429 [...] ``` Note that if we do not specify explicitly the remote-execution endpoint (via option `-r`), the tool will assume a remote-execution endpoint is exposed on the same socket (HOST:PORT pair). A mismatch in the remote-execution endpoint specified by the client and the one set up for the `just serve` instance will result in an error. ### Explicit interface, port or existing remote-execution endpoint There are good situations where relying on defaults is preferred. For example - the loopback device is a good option when the serve service runs on a dedicated machine; - relying on the default single-node remote-execution service means one can quickly start using the service on a shared machine with minimal effort, e.g., in a small team or for fast prototyping; - sometimes one does not know a fixed port number available for the desired serve endpoint. However, in most cases a remote-execution endpoint already exists and a finer control on the service endpoint is desired. The `just serve` configuration file can easily be set up with such information. For example, a serve service configured with ``` {.jsonc srcname=".just-servec"} { "remote service": { "interface": "127.0.0.1" , "port": 9999 } , "execution endpoint": {"address": "127.0.0.1:8989"} } ``` could then be exploited in a different shell as ``` sh $ just [...] -R 127.0.0.1:9999 -r 127.0.0.1:8989 # or $ just-mr -R 127.0.0.1:9999 -r 127.0.0.1:8989 [...] ``` In general, we recommend that the serve endpoint is _close_ to its associated remote-execution endpoint in a network sense (same subnet, or even same host, as above). More importantly, as with any service over the network, security should be considered, therefore next we will cover server and client certification, which ensure both the needed security, but also allows client access management. ### Enable mTLS As with `just execute`, mTLS must be enabled when the service starts, and it cannot be activated (or deactivated) while the serve service is running. The configuration file can be extended to specify both the client and server certification, such as ``` {.jsonc srcname=".just-servec"} ... , "remote-service": ... , "server cert": {"root": "system", "path": "etc/just-serve/certs/server.crt"} , "server key": {"root": "system", "path": "etc/just-serve/certs/server.key"} ... , "authentication": { "ca cert": {"root": "system", "path": "etc/just-serve/certs/ca.crt"} , "client cert": {"root": "system", "path": "etc/just-serve/certs/client.crt"} , "client key": {"root": "system", "path": "etc/just-serve/certs/client.key"} } ... ``` It must be noted that we expect the same `CA certificate` was used on the associated remote-execution endpoint, and naturally any client of the serve service should connect by passing the same `CA certificate` and a pair of certificate and private key signed by the same authority as the serve and execution endpoints. Therefore, a client using the serve service will connect as ``` sh $ just [...] -R [-r ] --tls-ca-cert \ --tls-client-cert --tls-client-key # or $ just-mr -R [-r ] --tls-ca-cert \ --tls-client-cert --tls-client-key [...] ``` Note that the serve configuration file requires location objects to be specified (which can be relative to the system root or to the current user's home directory), while the command line argument paths are expected to be either relative to the invocation directory or absolute. ### Known repositories A simple way to maintain the dependencies of a project is to store its list of needed third-party archives under version control (for our tool, a Git repository). For this purpose, `just serve` can be configured to be made aware at startup of local (with respect to the service environment) Git repositories checkout locations, places where the service can look for required Git objects, such as archives or repository roots. To do this, the configuration file can be extended, for example, with ``` {.jsonc srcname=".just-servec"} ... , "repositories": [ {"root": "system", "path": "var/repos/third-party-distfiles"} , {"root": "system", "path": "var/repos/project-foo"} , {"root": "system", "path": "var/repos/project-bar"} ] ... ``` ### Local build root In order to provide values of target-cache entries to the client, the serve service has to have its own target-level cache, and thus its own _local build root_. If one does not want to use the usual default location, the configuration file can be extended, for example, with ``` {.jsonc srcname=".just-servec"} ... , "local build root": {"root": "system", "path": "var/cache/serve-build-root"} ... ``` Do keep in mind that the build root must be given as an absolute path. Importantly, the local build root, as is generally the case for *justbuild*, is the only place where the tool will write. ### Info file To more easily handle the logged data provided by the running service, i.e., interface, pid, and port, we can configure the serve service to provide this information also in a file, which then can simply be parsed. To do this we can extend the `"remote service"` field accordingly ``` {.jsonc srcname=".just-servec"} ... , "remote-service": ... , "pid file": {"root": "system", "path": "var/run/info.json"} ... ... ``` Serving export targets ---------------------- For the following, we return to the *hello_world* example from the section on [*Building Third-party dependencies*](./third-party-software.md), which depends on the open-source project [fmtlib](https://github.com/fmtlib/fmt). Our `repos.json` at this stage reads: ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "fmt-targets-layer": { "repository": { "type": "file" , "path": "./fmt-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": { "type": "git" , "branch": "8.1.1" , "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9" , "repository": "https://github.com/fmtlib/fmt.git" } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } } ``` What we would like to do is use a `just serve` instance to provide the `"fmt"` export target, used to build the *hello_world* binary. In our final setup, the source code of the `fmtlib` library would never need to ever be fetched by any client building this project. To achieve this, in simple terms, the requirements needed to be met to build the `"fmt"` export target locally (as done so far) will need to be met by the serve endpoint. The following diagram showcases the distribution of roots between the client and the serve endpoint Locally . On the serve endpoint . . workspace_root ┌──────────┐ . ┌───────────────>│ "fmtlib" │ ┌─────────────────────┐ target_root . ┌───────┐ │ └──────────┘ │ "fmt-targets-layer" │<───────────────┤ "fmt" ├──┤ └─────────────────────┘ . └───────┘ │ bindings ┌────────────┐ . └───────────────>│ "rules-cc" │ . └────────────┘ What this diagram showcases is that for the `"fmt"` target - the source code is provided by repository `"fmtlib"`. This needs to be made available to the serve endpoint and a correct setup (as will be shown) can ensure that the client never has to fetch this repository locally. - the target description is provided by repository `"fmt-targets-layer"`, made content-defined by the `"to_git"` pragma and available locally in the Git checkout `./tutorial-defaults`. What makes `just serve` more than a simple dependency manager is that it allows for the target description to be changed locally and dispatch a build of a target without ever having to have the source code. - the repository `"rules-cc"` is needed on the serve endpoint as a transitive dependency, as it contains the rules by which the `"fmt"` target needs to be built. - finally, the build of target `"fmt"` is left to the serve endpoint, which will dispatch it to the respective remote-execution endpoint, with the client just receiving the target-level cache value and resulting artifacts. In the following we will showcase how a `just serve` instance can be configured for this scenario, together with the changes needed in the local `repos.json` build description in order to build a *hello_world* binary from a client that never has to fetch the source code of fmtlib. Let us start with a basic `.just-servec` configuration file, on which we can then expand step-by-step ``` {.jsonc srcname=".just-servec"} { "local build root": {"root": "system", "path": "var/cache/serve-build-root"} , "remote service": {"port": 9999} } ``` ### Serving third-party dependencies: "fmtlib" For building third-party dependencies from source the typical input is in the form of archived packages. This ensures easy auditing, code reproducibility, and long term offline availability for analysis and, of course, building. The `"fmtlib"` repository falls under this category, therefore in order to make it easily maintainable via a serve endpoint we need to switch its description to an archive-type repository. The fmtlib project offers for the tagged commit used by us so far an [official packaged version](https://github.com/fmtlib/fmt/releases/download/8.1.1/fmt-8.1.1.zip) (Git object hash: fd4144c2835f89516cac0db1f3c7b73562555dca), therefore the first step is to change repository `"fmtlib"` in our `repos.json` to type `"zip"`, populating its required fields as needed. Secondly, we can instruct *justbuild* to treat this repository as _absent_, which means that `just-mr` will set up this repository root such that it is not held locally, but it is known to the serve endpoint. This does not by itself exclude the possibility that the fmtlib archive will never be fetched, but, as will be shown below, pairing it with a good setup of the serve server will most certainly achieve this. With all these changes, the `"fmtlib"` entry in our `repos.json` now reads ``` {.jsonc srcname="repos.json"} ... , "fmtlib": { "repository": { "type": "zip" , "content": "fd4144c2835f89516cac0db1f3c7b73562555dca" , "fetch": "https://github.com/fmtlib/fmt/releases/download/8.1.1/fmt-8.1.1.zip" , "subdir": "fmt-8.1.1" , "pragma": {"absent": true} } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } ... ``` Now let us discuss what preparations are needed for the serve service in order to avoid clients having to fetch archives from the network. The typical way a project keeps track of their third-party packages is under some version control, e.g., a Git repository. Assuming the referenced fmtlib archive is available in such a Git repository, we can create a local checkout at path `/var/repos/distfiles`. Then the following command should pass ``` sh $ git -C /var/repos/distfiles cat-file -t fd4144c2835f89516cac0db1f3c7b73562555dca blob $ ``` This repository checkout can then be made available to the serve service by adding its path to the known repository list in the `.just-servec` configuration file, which will become ``` {.jsonc srcname=".just-servec"} { "local build root": {"root": "system", "path": "var/cache/serve-build-root"} , "remote service": {"port": 9999} , "repositories": [{"root": "system", "path": "var/repos/distfiles"}] } ``` Do keep in mind that deciding which Git checkout locations are known to the serve endpoint needs to be decided before the service is started, and no modification to its configuration can be done during its operation. Any updates would thus need a redeployment of the service. ### Serving first-party dependencies: "rules-cc" Projects in general can also depend on first-party code, which is more actively developed, such as code developed in the same organization by other teams, that is more closely coupled and more frequently updated than third-party code. For the most case, projects will want to use up-to-date verified versions of these codes, for example the latest commit of a Git repository which uses proper CI integration to test everything merged to its main branch. In our tutorial example, we will consider `"rules-cc"` as a first-party dependency. We will start with the serve service setup. As we want to be able to easily change the commit we are interested in whenever we expect the latest version of this repository in our project, on the serve server we will create a local Git checkout of `"rules-cc"` stored at `/var/repos/rules-cc`. This means that a client can easily update its build description to point to whichever commit it needs, while the serve server will have to ensure this checkout is always kept updated (usually automatized, for example via a cron job). The configuration file of the serve service will need thus to be updated to include this checkout location, so `.just-servec` now reads ``` {.jsonc srcname=".just-servec"} { "local build root": {"root": "system", "path": "var/cache/serve-build-root"} , "remote service": {"port": 9999} , "repositories": [ {"root": "system", "path": "var/repos/distfiles"} , {"root": "system", "path": "var/repos/rules-cc"} ] } ``` On the client side for our *hello_world* example we however cannot mark the `"rules-cc"` as absent. This is because while the `"fmt"` export target requiring this binding can be served, the main target of the tutorial is fully local, which requires `"rules-cc"` to be present. However, the information on which rule root is used is known, so during the dispatched build the serve endpoint will search for this root in its known repositories and find it in the prepared local checkout. ### Putting it all together We are now ready to see how this setup works. At this point the `repos.json` is ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "fmt-targets-layer": { "repository": { "type": "file" , "path": "./fmt-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": { "type": "zip" , "content": "fd4144c2835f89516cac0db1f3c7b73562555dca" , "fetch": "https://github.com/fmtlib/fmt/releases/download/8.1.1/fmt-8.1.1.zip" , "subdir": "fmt-8.1.1" , "pragma": {"absent": true} } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } } ``` and the `.just-servec` configuration file is ``` {.jsonc srcname=".just-servec"} { "local build root": {"root": "system", "path": "var/cache/serve-build-root"} , "remote service": {"port": 9999} , "repositories": [ {"root": "system", "path": "var/repos/distfiles"} , {"root": "system", "path": "var/repos/rules-cc"} ] } ``` We can now start the serve service ``` sh $ just serve .just-servec INFO: serve and execute services started: {"interface":"127.0.0.1","pid":4178555,"port":9999} ``` and, in a different shell, build *hello_world* in a clean build root using this serve endpoint ``` sh $ just-mr -R localhost:9999 --local-build-root ~/local-build-root build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","--local-build-root","/home/tutorial/local-build-root","-R","127.0.0.1:9999","helloworld"] INFO: Using '127.0.0.1:9999' as the remote execution endpoint. INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 0 cached, 1 served, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` We use a clean build root (here, in the `$HOME` directory of the current user, in our example `/home/tutorial/`) to show that we indeed not use any old cache entries from previous builds and that everything marked absent really comes from the serve endpoint. The running shell where the serve service was started in will have been updated to ``` log INFO: serve and execute services started: {"interface":"127.0.0.1","pid":4178555,"port":9999} INFO (target-service): Analysed target [["@","0","","fmt"],{"ADD_CXXFLAGS":null,"AR":null,"CXX":null,"CXXFLAGS":null,"ENV":null}] INFO (execution-service): Execute 62a33fc12031c240d38d12b183a47f79e9ce90ea58 INFO (execution-service): Execute 62af85caddbbc28caa08d98e004c6fa8772f69b057 INFO (execution-service): Execute 6233fc78dcdcde8fd3c4bdda7cbaf7f915d8c7a01b INFO (execution-service): Execute 62d181d81cac1e6a8c331c3eac643fb9ad0cac4cdd ``` showing that the `"fmt"` export target has been analysed and built by the serve endpoint (with the build dispatched to the associated remote-execution endpoint, which in our case coincides with the serve endpoint, hence the merged output information). ### Final notes For our example project we, of course, worked locally and deployed the serve endpoint on the loopback device. Those who want to further test this *hello_world* with a serve endpoint deployed actually on, e.g., a different physical or logical partition, to better simulate the client-server separation, would have to move to the new server deployment location just the `.just-servec` configuration file and the `repos` directory. Absent repositories and rc-files ---------------------------------- As covered in the above, once we have locally a correct build description separating first- and third-party dependencies, the only change made locally to request that repository roots be served by a serve endpoint is the addition of the pragma tag `{"absent": true}`. While in our example we only dealt with one absent repository as we had one export target, projects usually have tens of dependencies providing tens or hundreds of export targets, all of which could be managed and served by a suitably set up `just serve` instance. With different clients having different local and remote setups, it is preferred if one main build description is packaged with a project and its interaction with the various remote endpoints (serve and execution) can be easily maintained by the user. Thankfully, `just-mr` accepts as an argument a _repository configuration_ file (or _rc-file_, for short), which can hold not only all the usual command line arguments, but also extra arguments, such as one related to absent repositories. Let us show how this work. First, as stated, we clean up our `repos.json` by removing any `"absent"` pragma fields ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "fmt-targets-layer": { "repository": { "type": "file" , "path": "./fmt-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": { "type": "zip" , "content": "fd4144c2835f89516cac0db1f3c7b73562555dca" , "fetch": "https://github.com/fmtlib/fmt/releases/download/8.1.1/fmt-8.1.1.zip" , "subdir": "fmt-8.1.1" } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } } ``` One can easily check that this allows us to build locally again, with the note that, as we have changed the `"fmtlib"` repository type, we need a fetch of the archive. But, of course, we do ***not*** want to fetch this archive and build locally, just maintain a build description that ***could*** do so in the absence of a properly set up serve endpoint. Next we create a file `absent.json` with content ``` {.jsonc srcname="absent.json"} ["fmtlib"] ``` This has a single `JSON` list of the names of the repositories that should be marked as absent. We have chosen a descriptive file name, but one is free to choose another. Then we need to set up the _rc-file_ of `just-mr`. By default, `just-mr` always tries to read such a configuration file, with the default location being `$HOME/.just-mrrc`. Here we will pass this file explicitly, so we create file `rc-file` in the current directory with content ``` {.jsonc srcname="rc-file"} { "local build root": {"root": "home", "path": "local-build-root"} , "remote serve": {"address": "localhost:9999"} , "absent": [{"root": "workspace", "path": "absent.json"}] } ``` Let us break it down. The local build root is given using a construct we call a _location object_, specifying a path relative to some root type. In this case, we use root `"home"` to signal that this path is relative to the current user's `${HOME}` directory (e.g., in the example below `/home/tutorial/`). Next we specify the serve endpoint address, which is straight-forward. Lastly, we specify the list of repositories to be marked absent by giving the `absent.json` path relative to the workspace. This is because typically one would keep such a file in the project's tree, as it contains information pertinent to this particular project only. The presence of the `ROOT` file in the current directory ensures the file will be picked up by `just-mr`. With all this, we can rebuild, using this _rc-file_ and with the same serve endpoint still running, successfully ``` sh $ just-mr --rc rc-file build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","--local-build-root","/home/tutorial/local-build-root","-R","localhost:9999","helloworld"] INFO: Using 'localhost:9999' as the remote execution endpoint. INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 0 cached, 1 served, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 4 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` Lastly, it should be mentioned that the `just-mr` rc-file can be instructed to import configurations from other rc-files, overwriting (or expanding, for list-type fields) the values in the main one. This means that a minimal desired configuration can be shipped with the project, and users then can import it into their local rc-file (which possibly gets shared by several projects). If we assume file `shipped-rc-file` contains ``` {.jsonc srcname="rc-file"} { "remote serve": {"address": "127.0.0.1:9900"} , "remote execution": {"address": "127.0.0.1:8989"} } ``` then using a modified `rc-file` reading ``` {.jsonc srcname="rc-file"} { "local build root": {"root": "home", "path": "local-build-root"} , "remote serve": {"address": "localhost:9999"} , "absent": [{"root": "workspace", "path": "absent.json"}] , "rc files": [{"root": "workspace", "path": "shipped-rc-file"}] } ``` would be equivalent to the single combined configuration of ``` {.jsonc} { "local build root": {"root": "home", "path": "local-build-root"} , "remote serve": {"address": "127.0.0.1:9900"} , "absent": [{"root": "workspace", "path": "absent.json"}] , "remote execution": {"address": "127.0.0.1:8989"} } ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/lint.md000066400000000000000000000357431516554100600235330ustar00rootroot00000000000000# Running Linters It is common to run some form of linters over the code base. It is desirable to also use our build tool for this to have the benefit of parallel (or even remote) build and sound caching. Additionally, this also allows the lint tools to see the file layout as it occurs in the actual compile action, including generated files. Remember that even for source files this layout does not have to coincide with the layout of files in the source repository. Conveniently, our build rules have support for collecting the relevant information needed for linting built in. If a target is built with the configuration variable `LINT` set to a true value, lint information is provided for the transitive sources; as [third-party dependencies](third-party-software.md) are typically exported without `LINT` among the flexible variables, that naturally forms a boundary of the "own" code (to be linted, as opposed to third-party code). So, continuing the [third-party tutorial](third-party-software.md), we can obtain abstract nodes for our sources (`main.cpp`, `greet/greet.hpp`, `greet/greet.cpp`). ``` sh $ just-mr analyse -D '{"LINT": true}' --dump-nodes - INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","-D","{\"LINT\": true}","--dump-nodes","-"] INFO: Requested target is [["@","tutorial","","helloworld"],{"LINT":true}] INFO: Analysed target [["@","tutorial","","helloworld"],{"LINT":true}] INFO: Export targets found: 0 cached, 1 uncached, 0 not eligible for caching INFO: Result of target [["@","tutorial","","helloworld"],{"LINT":true}]: { "artifacts": { "helloworld": {"data":{"id":"1154ef311dff82653bc6a1a92bfc6152bc116cb86652b4b5218385fe39054391","path":"work/helloworld"},"type":"ACTION"} }, "provides": { "debug-hdrs": { }, "debug-srcs": { }, "dwarf-pkg": { }, "lint": [ {"id":"aec2651a9f3b869554e3a2e5cc3ab85d86610a79aae388a4f7839396dc2167d2","type":"NODE"}, {"id":"509e506f8d0c2ebe4fca63fe7cc528be17165bae464410ac5fba97a6ed92930d","type":"NODE"}, {"id":"cb3bd5f3934e199ddf5d43eae6866f8c6ba449f060213a244c93f0826c070c3f","type":"NODE"} ], "package": { "to_bin": true }, "run-libs": { } }, "runfiles": { } } INFO: Target nodes of target [["@","tutorial","","helloworld"],{"LINT":true}]: { "509e506f8d0c2ebe4fca63fe7cc528be17165bae464410ac5fba97a6ed92930d": { "result": { "artifact_stage": { "include": { "data": { "id": "a882ca13d51c70aa6b02d4996aa426ee6e21bf85881ec0f6dbe0a278f5a27b7b" }, "type": "TREE" }, "work/greet/greet.hpp": { "data": { "path": "greet/greet.hpp", "repository": "tutorial" }, "type": "LOCAL" } }, "provides": { "cmd": ["c++","-O2","-Wall","-I","work","-isystem","include","-E","work/greet/greet.hpp"], "direct deps artifact names": ["include/fmt","work/greet/greet.hpp"], "extra outs": [], "src": "work/greet/greet.hpp" }, "runfiles": { } }, "type": "VALUE_NODE" }, "aec2651a9f3b869554e3a2e5cc3ab85d86610a79aae388a4f7839396dc2167d2": { "result": { "artifact_stage": { "include": { "data": { "id": "29d6c7fef3c48a1a3b15edd770b33d073d5c9cd1e6e9fb22917831fcdd762ebb" }, "type": "TREE" }, "work/main.cpp": { "data": { "path": "main.cpp", "repository": "tutorial" }, "type": "LOCAL" } }, "provides": { "cmd": ["c++","-O2","-Wall","-I","work","-isystem","include","-c","work/main.cpp","-o","work/main.o"], "direct deps artifact names": ["include/greet/greet.hpp"], "extra outs": [], "src": "work/main.cpp" }, "runfiles": { } }, "type": "VALUE_NODE" }, "cb3bd5f3934e199ddf5d43eae6866f8c6ba449f060213a244c93f0826c070c3f": { "result": { "artifact_stage": { "include": { "data": { "id": "a882ca13d51c70aa6b02d4996aa426ee6e21bf85881ec0f6dbe0a278f5a27b7b" }, "type": "TREE" }, "work/greet/greet.cpp": { "data": { "path": "greet/greet.cpp", "repository": "tutorial" }, "type": "LOCAL" }, "work/greet/greet.hpp": { "data": { "path": "greet/greet.hpp", "repository": "tutorial" }, "type": "LOCAL" } }, "provides": { "cmd": ["c++","-O2","-Wall","-I","work","-isystem","include","-c","work/greet/greet.cpp","-o","work/greet/greet.o"], "direct deps artifact names": ["include/fmt","work/greet/greet.hpp"], "extra outs": [], "src": "work/greet/greet.cpp" }, "runfiles": { } }, "type": "VALUE_NODE" } } ``` We find the sources in correct staging, together with the respective compile command (or preprocessing, in case of headers) provided. The latter is important, to find the correct include files and to know the correct defines to be used. Of course, those abstract nodes are just an implementation detail and there is a rule to define linting for the collected sources. It takes two programs (targets consisting of a single artifact), - the `linter` for running the lint task on a single file, and - the `summarizer` for summarizing the lint results; additionally, arbitrary `config` data can be given to have config files available, but also to use a linter built from source. As for every rule, the details can be obtained with the `describe` subcommand. ``` sh $ just-mr --main rules-cc describe --rule lint targets INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","describe","-C","...","--rule","lint","targets"] | Run a given linter on the lint information provided by the given targets. ... Target fields - "linter" | Single artifact running the lint checks. | | This program is invoked with ... ``` Let's go through these programs we have to provide one by one. The first one is supposed to call the actual linter; as many linters, including `clang-tidy` which we use as an example, prefer to obtain the command information through a [compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html) there is actually some work to do, especially as the directory entry has to be an absolute path. We also move the configuration file `.clang-tidy` from the configuration directory (located in a directory given to us through the environment variable `CONFIG`) to the position expected by `clang-tidy`. ``` {.python srcname="run_clang_tidy.py"} #!/usr/bin/env python3 import json import os import shutil import subprocess import sys def dump_meta(src, cmd): OUT = os.environ.get("OUT") with open(os.path.join(OUT, "config.json"), "w") as f: json.dump({"src": src, "cmd": cmd}, f) def run_lint(src, cmd): dump_meta(src, cmd) config = os.environ.get("CONFIG") shutil.copyfile(os.path.join(config, ".clang-tidy"), ".clang-tidy") db = [ {"directory": os.getcwd(), "arguments": cmd, "file": src}] with open("compile_commands.json", "w") as f: json.dump(db,f) new_cmd = [ "clang-tidy", src ] return subprocess.run(new_cmd).returncode if __name__ == "__main__": sys.exit(run_lint(sys.argv[1], sys.argv[2:])) ``` The actual information on success or failure is provided through the exit code and information on the problems discovered (if any) is reported on stdout or stderr. Additionally, our launcher also writes the meta data in a file `config.json` in the directory for additional (usually machine-readable) diagnose output; the location of this directory is given to us by the environment variable `OUT`. We use a pretty simple `.clang-tidy` for demonstration purpose. ``` {.md srcname=".clang-tidy"} Checks: 'clang-analyzer-*,misc-*,-misc-include-*' WarningsAsErrors: 'clang-analyzer-*,misc-*,-misc-include-*' ``` Computing a summary of the individual lint results (given to the summarizer as subdirectories of the current working directory) is straight forward: the overall linting passed if all individual checks passed and for the failed tests we format stdout and stderr in some easy-to-read way; additionally, we also provide a machine-readable summary of the failures. ``` {.python srcname="summary.py"} #!/usr/bin/env python3 import json import os import sys FAILED = {} for lint in sorted(os.listdir()): if os.path.isdir(lint): with open(os.path.join(lint, "result")) as f: result = f.read().strip() if result != "PASS": record = {} with open(os.path.join(lint, "out/config.json")) as f: record["config"] = json.load(f) with open(os.path.join(lint, "stdout")) as f: log = f.read() with open(os.path.join(lint, "stderr")) as f: log += f.read() record["log"] = log FAILED[lint] = record with open(os.path.join(os.environ.get("OUT"), "failures.json"), "w") as f: json.dump(FAILED, f) failures = list(FAILED.keys()) for f in failures: src = FAILED[f]["config"]["src"] log = FAILED[f]["log"] print("%s %s" % (f, src)) print("".join([" " + line + "\n" for line in log.splitlines()])) if failures: sys.exit(1) ``` Of course, our launcher and summarizer have to be executable ``` sh $ chmod 755 run_clang_tidy.py summary.py ``` Now we can define our lint target. ``` {.jsonc srcname="TARGETS"} ... , "lint": { "type": ["@", "rules", "lint", "targets"] , "targets": ["helloworld"] , "linter": ["run_clang_tidy.py"] , "summarizer": ["summary.py"] , "config": [".clang-tidy"] } ... ``` As most rules, the lint rules also have a `"defaults"` target, which allows to set `PATH` appropriately for all lint actions. This can be useful if the linters are installed in a non-standard directory. ``` sh $ mkdir -p tutorial-defaults/lint $ echo '{"defaults": {"type": "defaults", "PATH": ["'"${TOOLCHAIN_PATH}"'"]}}' > tutorial-defaults/lint/TARGETS $ git add tutorial-defaults $ git commit -m 'add lint defaults' ``` We now can build our lint report in the same way as any test report. ``` sh $ just-mr build lint -P report INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","lint","-P","report"] INFO: Requested target is [["@","tutorial","","lint"],{}] INFO: Analysed target [["@","tutorial","","lint"],{}] INFO: Export targets found: 0 cached, 1 uncached, 0 not eligible for caching INFO: Target tainted ["lint"]. INFO: Discovered 11 actions, 0 tree overlays, 7 trees, 0 blobs INFO: Building [["@","tutorial","","lint"],{}]. INFO: Processed 7 actions, 3 cache hits. INFO: Artifacts built, logical paths are: out [a90a9e3a8ac23526eb31ae46c80434cfd5810ed5:41:t] report [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] work [52b9cfc07b53c59fb066bc95329f4ca6457e7338:111:t] INFO: Backing up artifacts of 1 export targets INFO: Target tainted ["lint"]. ``` Note the difference between the discovered and processed actions. This is inherent to the way linting works: in order to know the precise command line with which a source file is compiled, the respective target has to be analysed. However, to lint the source files, it is not necessary to actually build the binary whose source code should be linted. Of course, if a combined test target that includes linting as well as end-to-end tests is considered, the binary has to be built (and hence the actions processed) for other reasons. To see that some real linting is going on, let's modify one of our source files. Say, we'll make the greeting independent of the recipient. ``` {.cpp srcname="greet/greet.cpp"} #include "greet.hpp" #include void greet(std::string const& s) { fmt::print("Hello!\n"); } ``` Building succeeds without any warning. ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 1 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 1 cache hits. INFO: Artifacts built, logical paths are: helloworld [2cb87c743e9fd3d18543732945df3ef9ca084be6:132736:x] ``` However, the linter reports it. ``` sh $ just-mr build lint -P report || : INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","lint","-P","report"] INFO: Requested target is [["@","tutorial","","lint"],{}] INFO: Analysed target [["@","tutorial","","lint"],{}] INFO: Export targets found: 1 cached, 0 uncached, 0 not eligible for caching INFO: Target tainted ["lint"]. INFO: Discovered 8 actions, 0 tree overlays, 6 trees, 0 blobs INFO: Building [["@","tutorial","","lint"],{}]. WARN (action:415a94f0c74ec937e2504b9ef5f94696232ff2c57eb2bec00c226896e2eb8be6): lint failed for work/greet/greet.cpp (exit code 1); outputs: - "out" [caf25f0a518d21909625f9a7974002796f6d8b5f:39:t] - "result" [94e1707e853c36f514de3876408c09a0e0ca6fc4:5:f] - "stderr" [ffc377e8898697782ab96419f6ab82c60985c752:235:f] - "stdout" [17975e013bd2cc3f66509e11737e1e169f1bd162:231:f] INFO: Processed 4 actions, 2 cache hits. INFO: Artifacts built, logical paths are: out [c298959107421711f8d87a2b96e95858c065b9b9:41:t] FAILED report [0b0ab9eb90c28ece0f14a13a6ae5c97da4a32170:531:f] FAILED result [94e1707e853c36f514de3876408c09a0e0ca6fc4:5:f] FAILED work [007eec6bad8b691c067dd2c54165ac2912711474:111:t] FAILED INFO: Failed artifacts: out [c298959107421711f8d87a2b96e95858c065b9b9:41:t] FAILED report [0b0ab9eb90c28ece0f14a13a6ae5c97da4a32170:531:f] FAILED result [94e1707e853c36f514de3876408c09a0e0ca6fc4:5:f] FAILED work [007eec6bad8b691c067dd2c54165ac2912711474:111:t] FAILED 0000000002 work/greet/greet.cpp work/greet/greet.cpp:4:31: error: parameter 's' is unused [misc-unused-parameters,-warnings-as-errors] 4 | void greet(std::string const& s) { | ^ | /*s*/ 287 warnings generated. Suppressed 286 warnings (286 in non-user code). Use -header-filter=.* to display errors from all non-system headers. Use -system-headers to display errors from system headers as well. 1 warning treated as error INFO: Target tainted ["lint"]. WARN: Build result contains failed artifacts. ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/proto.md000066400000000000000000000716131516554100600237240ustar00rootroot00000000000000Using protocol buffers ====================== The rules *justbuild* uses for itself also support protocol buffers. This tutorial shows how to use those rules and the targets associated with them. It is not a tutorial on protocol buffers itself; rather, it is assumed that the reader has some knowledge on [protocol buffers](https://developers.google.com/protocol-buffers/). Setting up the repository configuration --------------------------------------- Before we begin, we first need to declare where the root of our workspace is located by creating the empty file `ROOT`: ``` sh $ touch ROOT ``` The `protobuf` repository conveniently contains an [example](https://github.com/protocolbuffers/protobuf/tree/v3.12.4/examples), so we can use this and just add our own target files. We create file `repos.template.json` as follows. ``` {.jsonc srcname="repos.template.json"} { "repositories": { "": { "repository": { "type": "zip" , "content": "7af7165b585e4aed714555a747b6822376176ef4" , "fetch": "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.12.4.zip" , "subdir": "protobuf-3.12.4/examples" } , "target_root": "tutorial" , "bindings": {"rules": "rules-cc"} } , "tutorial": {"repository": {"type": "file", "path": "."}} } } ``` The missing entry `"rules-cc"` refers to our C/C++ build rules provided [online](https://github.com/just-buildsystem/rules-cc). These rules support protobuf if the dependency `"protoc"` is provided. To import this rule repository including the required transitive dependencies for protobuf, the `just-import-git` script of the *justbuild* project can be used with option `--as rules-cc` to generate the actual `repos.json`: ``` sh $ just-import-git -C repos.template.json -b master --as rules-cc https://github.com/just-buildsystem/rules-cc > repos.json ``` To build the example with `just`, the only task is to write targets files. As that contains a couple of new concepts, we will do this step by step. The proto library ----------------- First, we have to declare the proto library. In this case, it only contains the file `addressbook.proto` and has no dependencies. To declare the library, create a `TARGETS` file with the following content: ``` {.jsonc srcname="TARGETS"} { "address": { "type": ["@", "rules", "proto", "library"] , "name": ["addressbook"] , "srcs": ["addressbook.proto"] } } ``` In general, proto libraries could also depend on other proto libraries; those would be added to the `"deps"` field. When building the library, there's very little to do after `just-mr` fetches and sets up all the repositories. ``` sh $ just-mr build address INFO: Performing repositories setup INFO: Found 23 repositories involved PROG: [ 44%] 0 computed, 1 local, 13 cached, 4 done; 7 fetches ("rules-cc/just/defaults", ...) PROG: [ 66%] 0 computed, 1 local, 13 cached, 6 done; 7 fetches ("rules-cc/just/defaults", ...) PROG: [ 66%] 0 computed, 1 local, 13 cached, 6 done; 7 fetches ("rules-cc/just/defaults", ...) PROG: [ 77%] 0 computed, 1 local, 13 cached, 7 done; 7 fetches ("rules-cc/just/defaults", ...) INFO: Setup finished, exec ["just","build","-C","...","--local-build-root","/tmp/proto","address"] INFO: Requested target is [["@","","","address"],{}] INFO: Analysed target [["@","","","address"],{}] INFO: Discovered 0 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","address"],{}]. INFO: Processed 0 actions, 0 cache hits. INFO: Artifacts built, logical paths are: $ ``` On the other hand, what did we expect? A proto library is an abstract description of a protocol, so, as long as we don't specify for which language we want to have bindings, there is nothing to generate. Nevertheless, a proto library target is not empty. In fact, it can't be empty, as other targets can only access the values of a target and have no insights into its definitions. We already relied on this design principle implicitly, when we exploited target-level caching for our external dependencies and did not even construct the dependency graph for that target. A proto library simply provides the dependency structure of the `.proto` files. ``` sh $ just-mr analyse --dump-nodes - address INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","--dump-nodes","-","address"] INFO: Requested target is [["@","","","address"],{}] INFO: Result of target [["@","","","address"],{}]: { "artifacts": { }, "provides": { "proto": [ {"id":"6bcfb07e77f4d00f84d4c38bff64b92e0a1cf07399bd0987250eaef1b06b0b50","type":"NODE"} ] }, "runfiles": { } } INFO: Target nodes of target [["@","","","address"],{}]: { "6bcfb07e77f4d00f84d4c38bff64b92e0a1cf07399bd0987250eaef1b06b0b50": { "node_type": "library", "string_fields": { "name": ["addressbook"], "stage": [""] }, "target_fields": { "deps": [], "srcs": [{"id":"dd79fcd0043ad155b5765f6a7a58a6c88fbcd38567ab0523684bd54a925727ce","type":"NODE"}] }, "type": "ABSTRACT_NODE" }, "dd79fcd0043ad155b5765f6a7a58a6c88fbcd38567ab0523684bd54a925727ce": { "result": { "artifact_stage": { "addressbook.proto": { "data": { "file_type": "f", "id": "b4b33b4c658924f0321ab4e7a9dc9cf8da1acec3", "size": 1234 }, "type": "KNOWN" } }, "provides": { }, "runfiles": { } }, "type": "VALUE_NODE" } } $ ``` The target has one provider `"proto"`, which is a node. Nodes are an abstract representation of a target graph. More precisely, there are two kind of nodes, and our example contains one of each. The simple kind of nodes are the value nodes; they represent a target that has a fixed value, and hence are given by artifacts, runfiles, and provided data. In our case, we have one value node, the one for the `.proto` file. The other kind of nodes are the abstract nodes. They describe the arguments for a target, but only have an abstract name (i.e., a string) for the rule. Combining such an abstract target with a binding for the abstract rule names gives a concrete "anonymous" target that, in our case, will generate the library with the bindings for the concrete language. In this example, the abstract name is `"library"`. The alternative in our proto rules would have been `"service library"`, for proto libraries that also contain `rpc` definitions (which is used by [gRPC](https://grpc.io/)). Using proto libraries --------------------- Using proto libraries requires, as discussed, bindings for the abstract names. Fortunately, our `CC` rules are aware of proto libraries, so we can simply use them. Our target file hence continues as follows. ``` {.jsonc srcname="TARGETS"} ... , "add_person": { "type": ["@", "rules", "CC", "binary"] , "name": ["add_person"] , "srcs": ["add_person.cc"] , "private-proto": ["address"] } , "list_people": { "type": ["@", "rules", "CC", "binary"] , "name": ["list_people"] , "srcs": ["list_people.cc"] , "private-proto": ["address"] } ... ``` The first time, we build a target that requires the proto compiler (in that particular version, built in that particular way), it takes a bit of time, as the proto compiler has to be built. But in follow-up builds, also in different projects, the target-level cache is filled already. ``` sh $ just-mr build add_person [...] $ just-mr build add_person INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","build","-C","...","add_person"] INFO: Requested target is [["@","","","add_person"],{}] INFO: Analysed target [["@","","","add_person"],{}] INFO: Export targets found: 2 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 5 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","","","add_person"],{}]. INFO: Processed 5 actions, 5 cache hits. INFO: Artifacts built, logical paths are: add_person [bca89ed8465e81c629d689b66c71deca138e2c27:2847912:x] $ ``` If we look at the actions associated with the binary, we find that those are still the two actions we expect: a compile action and a link action. (Some of the fields have been removed in the following example outputs and replaced by `"..."` for clarity.) ``` sh $ just-mr analyse add_person --dump-actions - INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","add_person","--dump-actions","-"] INFO: Requested target is [["@","","","add_person"],{}] INFO: Result of target [["@","","","add_person"],{}]: { "artifacts": { "add_person": {"data":{"id":"cb403cfeb7af26f83cb268056847f465d330ac44f7a563788305436b2640df2e","path":"work/add_person"},"type":"ACTION"} }, "provides": { "debug-hdrs": { }, "debug-srcs": { }, "dwarf-pkg": { }, "lint": [ ], "package": { "to_bin": true }, "run-libs": { } }, "runfiles": { } } INFO: Actions for target [["@","","","add_person"],{}]: [ { "command": ["c++","-O2",...,"-I","work","-isystem","include","-c","work/add_person.cc","-o","work/add_person.o"], "env": { "PATH": "/bin:/sbin:/usr/bin:/usr/sbin" }, "input": { ... }, "output": ["work/add_person.o"] }, { "command": ["c++","-Wl,-rpath,$ORIGIN","-Wl,-rpath,$ORIGIN/../lib","-o","add_person","-O2",...,"add_person.o","libaddressbook.a","libprotobuf.a","libprotobuf-lite.a",...,"libz.a"], "env": { "PATH": "/bin:/sbin:/usr/bin:/usr/sbin" }, "input": { ... }, "output": ["add_person"] } ] $ ``` As discussed, the `libaddressbook.a` that is conveniently available during the linking of the binary (as well as the `addressbook.pb.h` available in the `include` tree for the compile action) are generated by an anonymous target. Using that during the build we already filled the target-level cache, we can have a look at all targets still analysed. In the one anonymous target, we find again the abstract node we discussed earlier. ``` sh $ just-mr analyse add_person --dump-targets - INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","add_person","--dump-targets","-"] INFO: Requested target is [["@","","","add_person"],{}] INFO: Result of target [["@","","","add_person"],{}]: { "artifacts": { "add_person": {"data":{"id":"cb403cfeb7af26f83cb268056847f465d330ac44f7a563788305436b2640df2e","path":"work/add_person"},"type":"ACTION"} }, "provides": { "debug-hdrs": { }, "debug-srcs": { }, "dwarf-pkg": { }, "lint": [ ], "package": { "to_bin": true }, "run-libs": { } }, "runfiles": { } } INFO: List of analysed targets: { "#": { "c4f68b96f739e96f894c5b498ab2b3f0bc62df120419094f867b1d5769f5e4fa": { "6bcfb07e77f4d00f84d4c38bff64b92e0a1cf07399bd0987250eaef1b06b0b50": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "@": { "": { "": { "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"AR":null,"ARCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "address": [{}] } }, "rules-cc": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] }, "CC/proto": { "defaults": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "rules-cc/just/protobuf": { "": { "installed protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "libprotobuf": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "toolchain": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "toolchain_headers": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "rules-cc/just/rules": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "rules-cc/just/toolchain": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "unknown": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"TARGET_ARCH":null}] } } } } $ ``` It should be noted, however, that this tight integration of proto into our `C++` rules is just convenience of our code base. If we had to cooperate with rules not aware of proto, we could have created a separate rule delegating the library creation to the anonymous target and then simply reflecting the values of that target. In fact, we could simply use an empty library with a public `proto` dependency for this purpose. ``` {.jsonc srcname="TARGETS"} ... , "address proto library": {"type": ["@", "rules", "CC", "library"], "proto": ["address"]} ... ``` ``` sh $ just-mr analyse 'address proto library' INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","address proto library"] INFO: Requested target is [["@","","","address proto library"],{}] INFO: Result of target [["@","","","address proto library"],{}]: { "artifacts": { }, "provides": { ... "compile-deps": { ... "addressbook.pb.h": {"data":{"id":"bab5472beed7f032000d2d2cbf7d772c22c8c95cccdecbe70e73e9494ee34bf0","path":"work/addressbook.pb.h"},"type":"ACTION"}, ... }, ... "link-args": [ "libaddressbook.a", ... ], "link-deps": { ... "libaddressbook.a": {"data":{"id":"fa60ded058e1fc7663b4b925a1274d1fe330bedb12c8785a7042e450ef17a7c1","path":"work/libaddressbook.a"},"type":"ACTION"}, ... }, ... }, "runfiles": { } } $ ``` Adding a test ------------- Now, let's add a test. As we use the `protobuf` repository as workspace root, we add the test script ad hoc into a targets file, using the `"file_gen"` rule. For debugging a potentially failing test, we also keep the intermediate files the test generates. Add to the top-level `TARGETS` file the following content: ``` {.jsonc srcname="TARGETS"} ... , "test.sh": { "type": "file_gen" , "name": "test.sh" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -e" , "(echo 12345; echo 'John Doe'; echo 'jdoe@example.org'; echo) | ./add_person addressbook.data" , "./list_people addressbook.data > out.txt" , "grep Doe out.txt" ] } } , "test": { "type": ["@", "rules", "shell/test", "script"] , "name": ["read-write-test"] , "test": ["test.sh"] , "deps": ["add_person", "list_people"] , "keep": ["addressbook.data", "out.txt"] } ... ``` That example also shows why it is important that the generation of the language bindings is delegated to an anonymous target: we want to analyse only once how the `C++` bindings are generated. Nevertheless, many targets can depend (directly or indirectly) on the same proto library. And, indeed, analysing the test, we get the expected additional targets and the one anonymous target is reused by both binaries. ``` sh $ just-mr analyse test --dump-targets - INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","test","--dump-targets","-"] INFO: Requested target is [["@","","","test"],{}] INFO: Result of target [["@","","","test"],{}]: { "artifacts": { "pwd": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"pwd"},"type":"ACTION"}, "result": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"result"},"type":"ACTION"}, "stderr": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"stderr"},"type":"ACTION"}, "stdout": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"stdout"},"type":"ACTION"}, "time-start": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"time-start"},"type":"ACTION"}, "time-stop": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"time-stop"},"type":"ACTION"}, "work/addressbook.data": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"work/addressbook.data"},"type":"ACTION"}, "work/out.txt": {"data":{"id":"73744cdcd8be5082a6960ec0cf8c929d2a9cd0dd65860e3458901c49c8e4744f","path":"work/out.txt"},"type":"ACTION"} }, "provides": { "lint": [ ] }, "runfiles": { "read-write-test": {"data":{"id":"89bfa89b538549aa812415be625759872191d05315726828829848a87f86fa9f"},"type":"TREE"} } } INFO: List of analysed targets: { "#": { "c4f68b96f739e96f894c5b498ab2b3f0bc62df120419094f867b1d5769f5e4fa": { "6bcfb07e77f4d00f84d4c38bff64b92e0a1cf07399bd0987250eaef1b06b0b50": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "@": { "": { "": { "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"AR":null,"ARCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "address": [{}], "list_people": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"AR":null,"ARCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "test": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"AR":null,"ARCH":null,"ARCH_DISPATCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"DWP":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"LINT":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"RUNS_PER_TEST":null,"TARGET_ARCH":null,"TEST_ENV":null,"TEST_SUMMARY_EXECUTION_PROPERTIES":null,"TIMEOUT_SCALE":null,"TOOLCHAIN_CONFIG":null}], "test.sh": [{}] } }, "rules-cc": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] }, "CC/proto": { "defaults": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] }, "shell": { "defaults": [{"ARCH":null,"HOST_ARCH":null,"TARGET_ARCH":null}] } }, "rules-cc/just/protobuf": { "": { "installed protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "libprotobuf": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "toolchain": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"PREFIX":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "toolchain_headers": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"PKG_CONFIG_ARGS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "rules-cc/just/rules": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "rules-cc/just/toolchain": { "CC": { "defaults": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "unknown": [{"ARCH":null,"DEBUG":null,"HOST_ARCH":null,"TARGET_ARCH":null}] } } } } INFO: Target tainted ["test"]. $ ``` Finally, the test passes and the output is as expected. ``` sh $ just-mr build test -P work/out.txt INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","build","-C","...","test","-P","work/out.txt"] INFO: Requested target is [["@","","","test"],{}] INFO: Analysed target [["@","","","test"],{}] INFO: Export targets found: 2 cached, 0 uncached, 0 not eligible for caching INFO: Target tainted ["test"]. INFO: Discovered 8 actions, 0 tree overlays, 5 trees, 1 blobs INFO: Building [["@","","","test"],{}]. INFO: Processed 8 actions, 5 cache hits. INFO: Artifacts built, logical paths are: pwd [9006e78b54c8f3118918d2d471c79745ffa0c8a0:311:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] stderr [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] stdout [7fab9dd1ee66a1e76a3697a27524f905600afbd0:196:f] time-start [0836c60e080d14176345259e32f65c1e14edfc49:11:f] time-stop [0836c60e080d14176345259e32f65c1e14edfc49:11:f] work/addressbook.data [036614cb6ec77ecf979729cbef3f22bd5ebd6a56:41:f] work/out.txt [90ffe98823f628b92fe54b8c85f1754a9085c8db:101:f] (1 runfiles omitted.) Person ID: 12345 Name: John Doe E-mail address: jdoe@example.org Updated: ... INFO: Target tainted ["test"]. $ ``` Debugging with generated files ------------------------------ Finally, let's look at how debugging using protobufs looks like. In the root `TARGETS` file we can add a debug version of the `"list_people"` binary, together with the target to stage its respective debug artifacts (as described in the chapter on [*Debugging*](./hello-world.md)). ``` {.jsonc srcname="TARGETS"} ... , "list_people debug": { "type": "configure" , "target": "list_people" , "config": {"type": "'", "$1": {"DEBUG": {"USE_DEBUG_FISSION": false}}} } , "list_people debug staged": { "type": ["@", "rules", "CC", "install-with-deps"] , "targets": ["list_people debug"] } ... ``` The debugging target can then be installed to a directory of our choosing. ``` sh $ just-mr install "list_people debug staged" -o .ext/debug INFO: Performing repositories setup INFO: Found 23 repositories involved INFO: Setup finished, exec ["just","install","-C","...","list_people debug staged","-o",".ext/debug"] INFO: Requested target is [["@","","","list_people debug staged"],{}] INFO: Analysed target [["@","","","list_people debug staged"],{}] INFO: Export targets found: 0 cached, 98 uncached, 0 not eligible for caching INFO: Discovered 447 actions, 0 tree overlays, 99 trees, 1 blobs INFO: Building [["@","","","list_people debug staged"],{}]. [...] INFO: Processed 447 actions, 1 cache hits. INFO: Artifacts can be found in: /tmp/tutorial/.ext/debug/bin/list_people [71165d3aed4491169ed8bb8e783a9bbc15b9e9b6:46902576:x] /tmp/tutorial/.ext/debug/include/absl/algorithm/algorithm.h [59aeed7d264d927e44648428056cc9c489fad844:2190:f] /tmp/tutorial/.ext/debug/include/absl/algorithm/container.h [6bbe3b5adf40b8a4bc48551f8e9f0af96dd98ace:80031:f] ... /tmp/tutorial/.ext/debug/include/absl/utility/utility.h [ebbb49b7159a3bcf2f1edbf9d44bd67dceb0329f:7646:f] /tmp/tutorial/.ext/debug/include/addressbook.pb.h [f273099fe8d8345b1e61bdfa19751aefebe55869:46787:f] /tmp/tutorial/.ext/debug/include/google/protobuf/any.h [fe8a1775c77b91fc08368833d70236509dde82b7:6127:f] /tmp/tutorial/.ext/debug/include/google/protobuf/any.pb.h [67185ebca130f840ddacd687c59faceb523730e4:16787:f] /tmp/tutorial/.ext/debug/include/google/protobuf/any.proto [eff44e5099da27f7fb1ef14bb34902ccf4250b89:6154:f] ... /tmp/tutorial/.ext/debug/include/google/protobuf/wrappers.pb.h [0d85637aa1dadd86ccbc81a9563d54aba9eb9b3b:77623:f] /tmp/tutorial/.ext/debug/include/google/protobuf/wrappers.proto [1959fa55a4e7f284a9d6a78a447c5d89d137e87c:4044:f] /tmp/tutorial/.ext/debug/include/utf8_range.h [d7c232616022bb28c8bc35ca62d61f890a2f8ced:540:f] /tmp/tutorial/.ext/debug/include/utf8_validity.h [1f251d0fec0a2ae496335780d93a2869a1d7f743:712:f] /tmp/tutorial/.ext/debug/include/zconf.h [62adc8d8431f2f9149ae0b1583915e21a28dd8b5:16500:f] /tmp/tutorial/.ext/debug/include/zlib.h [8d4b932eaf6a0fbb8133b3ab49ba5ef587059fa0:96829:f] /tmp/tutorial/.ext/debug/work/absl/algorithm/algorithm.h [59aeed7d264d927e44648428056cc9c489fad844:2190:f] /tmp/tutorial/.ext/debug/work/absl/algorithm/container.h [6bbe3b5adf40b8a4bc48551f8e9f0af96dd98ace:80031:f] ... /tmp/tutorial/.ext/debug/work/absl/utility/utility.h [ebbb49b7159a3bcf2f1edbf9d44bd67dceb0329f:7646:f] /tmp/tutorial/.ext/debug/work/addressbook.pb.cc [2261b2349a90b090d501c3c3c362e1efc783c218:46574:f] /tmp/tutorial/.ext/debug/work/addressbook.pb.h [f273099fe8d8345b1e61bdfa19751aefebe55869:46787:f] ... /tmp/tutorial/.ext/debug/work/google/protobuf/any.h [fe8a1775c77b91fc08368833d70236509dde82b7:6127:f] /tmp/tutorial/.ext/debug/work/google/protobuf/any.pb.h [67185ebca130f840ddacd687c59faceb523730e4:16787:f] /tmp/tutorial/.ext/debug/work/google/protobuf/any.proto [eff44e5099da27f7fb1ef14bb34902ccf4250b89:6154:f] ... /tmp/tutorial/.ext/debug/work/list_people.cc [b309c596804739007510f06ff62c12898275fec7:2268:f] ... /tmp/tutorial/.ext/debug/work/zlib.h [8d4b932eaf6a0fbb8133b3ab49ba5ef587059fa0:96829:f] /tmp/tutorial/.ext/debug/work/zutil.c [b1c5d2d3c6daf5a4b7a337dafe3e862ca177b41c:7179:f] /tmp/tutorial/.ext/debug/work/zutil.h [48dd7febae65eeeaad7794f0a9317bcd054c107f:6677:f] INFO: Backing up artifacts of 98 export targets ``` As the command requires (re)building many targets in debug mode, some of the resulting output was replaced with `"..."` for brevity. What needs to be noted here is that for each (directly or transitively) included proto file the corresponding _generated_ `.pb.h` header file and `.pb.cc` source file get staged as well, ensuring that also all needed proto symbols will be available to a debugger. In order to debug now this binary, one can, for example, use the `addressbook.data` file generated by the previous test, staged accordingly, as input database for the `list_people` binary. ``` sh $ just-mr install test -o .ext/test_data [...] $ cd .ext/debug $ gdb --args bin/list_people ../test_data/work/addressbook.data ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/rebuild.md000066400000000000000000000225631516554100600242070ustar00rootroot00000000000000Ensuring reproducibility of the build ===================================== Software builds should be [reproducible](https://reproducible-builds.org/). *Justbuild* supports this goal in local builds by isolating individual actions, setting permissions and file time stamps to canonical values, etc.; most remote execution systems take even further measures to ensure the environment always looks the same to every action. Nevertheless, it is always possible to break reproducibility by bad actions, both coming from rules not carefully written, as well as from ad-hoc actions added by the `generic` target, such as ``` jsonc ... , "version.h": { "type": "generic" , "cmds": ["echo '#define VERSION \"0.0.0.'`date +%Y%m%d%H%M%S`'\"' > version.h"] , "outs": ["version.h"] } ... ``` Besides time stamps there are many other sources of nondeterminism, like properties of the build machine (name, number of CPUs available, etc), but also subtle ones like `readdir` order. Often, those non-reproducible parts get buried deeply in a final artifact (like the version string embedded in a binary contained in a compressed installation archive); and, as long as the non-reproducible action stays in cache, it does not even result in bad incrementality. Still, others won't be able to reproduce the exact artifact. There are tools like [diffoscope](https://diffoscope.org/) to deeply compare archives and other container formats. Nevertheless, it is desirable to find the root causes, i.e., the first (in topological order) actions that yield a different output. Rebuilding ---------- For the remainder of this section, we will consider the following example project with the C++ source file `hello.cpp`: ``` {.cpp srcname="hello.cpp"} #include #include "version.h" int main(int argc, const char* argv[]) { if (argc > 1 && std::string{argv[1]} == "-v") { std::cout << VERSION << std::endl; } std::cout << "Hello world!\n"; return 0; } ``` and the following `TARGETS` file: ``` {.jsonc srcname="TARGETS"} { "": { "type": "install" , "files": { "bin/hello": "hello" , "share/hello/version.txt": "version.txt" , "share/hello/OUT.txt": "OUT.txt" } } , "hello": { "type": ["@", "rules", "CC", "binary"] , "name": ["hello"] , "srcs": ["hello.cpp"] , "private-hdrs": ["version.h"] } , "version.h": { "type": "generic" , "cmds": ["echo '#define VERSION \"0.0.0.'`date +%Y%m%d%H%M%S`'\"' > version.h"] , "outs": ["version.h"] } , "version.txt": { "type": "generic" , "outs": ["version.txt"] , "cmds": ["./hello -v > version.txt"] , "deps": ["hello"] } , "out.txt": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["./hello > out.txt"] , "deps": ["hello"] } , "OUT.txt": { "type": "generic" , "outs": ["OUT.txt"] , "cmds": ["tr a-z A-Z > OUT.txt < out.txt"] , "deps": ["out.txt"] } } ``` The `repos.json` only needs the `"rules-cc"` repository and as main repository the current working directory ``` {.jsonc srcname="repos.json"} { "main": "" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } } , "": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc"} } } } ``` To search for the root cause of non-reproducibility, `just` has a subcommand `rebuild`. It builds the specified target again, requesting that every action be executed again (but target-level cache is still active); then the result of every action is compared to the one in the action cache, if present with the same inputs. So, you typically would first `build` and then `rebuild`. Note that a repeated `build` simply takes the action result from cache. ``` sh $ touch ROOT $ just-mr build INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","build","-C","..."] INFO: Requested target is [["@","","",""],{}] INFO: Analysed target [["@","","",""],{}] INFO: Discovered 6 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","","",""],{}]. INFO: Processed 6 actions, 0 cache hits. INFO: Artifacts built, logical paths are: bin/hello [1910a58cdd5c270ca375b3222ec1e602b00dee73:18072:x] share/hello/OUT.txt [428b97b82b6c59cad7488b24e6b618ebbcd819bc:13:f] share/hello/version.txt [de0d4f12aeb65c9e0a52909a07b0638e16e112fd:34:f] $ sleep 1 $ just-mr build INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","build","-C","..."] INFO: Requested target is [["@","","",""],{}] INFO: Analysed target [["@","","",""],{}] INFO: Discovered 6 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","","",""],{}]. INFO: Processed 6 actions, 6 cache hits. INFO: Artifacts built, logical paths are: bin/hello [1910a58cdd5c270ca375b3222ec1e602b00dee73:18072:x] share/hello/OUT.txt [428b97b82b6c59cad7488b24e6b618ebbcd819bc:13:f] share/hello/version.txt [de0d4f12aeb65c9e0a52909a07b0638e16e112fd:34:f] $ just-mr rebuild INFO: Performing repositories setup INFO: Found 2 repositories involved INFO: Setup finished, exec ["just","rebuild","-C","..."] INFO: Requested target is [["@","","",""],{}] INFO: Analysed target [["@","","",""],{}] INFO: Discovered 6 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Rebuilding [["@","","",""],{}]. WARN: Found flaky action: - id: 50e387d4d4c4dd9d8e6d08e1895c7dc729e5a4f3e7c7ad90cc93e373b5dea947 - cmd: ["sh","-c","echo '#define VERSION \"0.0.0.'`date +%Y%m%d%H%M%S`'\"' > version.h\n"] - output 'version.h' differs: - [a3c9ccb6547a898c51c2d46cb651f2df668ef007:39:f] (rebuilt) - [d8a442743402f7b589e2c25f7981149eeaa1a8f8:39:f] (cached) INFO: 2 actions compared with cache, 1 flaky actions found (0 of which tainted), no cache entry found for 4 actions. INFO: Artifacts built, logical paths are: bin/hello [84d0282a5b1a9ab09638d02955ad1e92aa911103:18072:x] share/hello/OUT.txt [428b97b82b6c59cad7488b24e6b618ebbcd819bc:13:f] share/hello/version.txt [d15119f103c0c1322e759c5e9fe5ef45926036fa:34:f] $ ``` In the example, the second action compared to cache is the upper casing of the output. Even though the generation of `out.txt` depends on the non-reproducible `hello`, the file itself is reproducible. Therefore, the follow-up actions are checked as well. For this simple example, reading the console output is enough to understand what's going on. However, checking for reproducibility usually is part of a larger, quality-assurance process. To support the automation of such processes, the findings can also be reported in machine-readable form. ``` sh $ just-mr rebuild --dump-flaky flakes.json --dump-graph actions.json [...] $ cat flakes.json { "cache misses": [ "059fc6b8047bbaf6353f5813be72e387406dd9a171da1f628b167785ed710f84", "d2ae0c3a1b3e588e531ff9624def1dbddff9e61b185888602704854f2ab6338d", "1c7636801667a48bbb0fbd5fa5404dbff32d92150a6d6fb54b8d48f9ca648271", "8ae961996bd2c4c03afb29549053dc9a9cd8d0cc12a0e58aade87159e133c528" ], "flaky actions": { "50e387d4d4c4dd9d8e6d08e1895c7dc729e5a4f3e7c7ad90cc93e373b5dea947": { "version.h": { "cached": { "file_type": "f", "id": "d8a442743402f7b589e2c25f7981149eeaa1a8f8", "size": 39 }, "rebuilt": { "file_type": "f", "id": "6fe7020f82b32335ee3478e8f7628e293c995139", "size": 39 } } } } }$ ``` The file reports the flaky actions together with the non-reproducible artifacts they generated, reporting both, the cached and the newly generated output. The files themselves can be obtained via `just install-cas` as usual, allowing deeper comparison of the outputs. The full definitions of the actions can be found in the action graph, in the example dumped as well as `actions.json`; this definition also includes the origins for each action, i.e., the configured targets that requested the respective action. Comparing build environments ---------------------------- Simply rebuilding on the same machine is good way to detect embedded time stamps of sufficiently small granularity; for other sources of non-reproducibility, however, more modifications of the environment are necessary. A simple, but effective, way for modifying the build environment is the option `-L` to set the local launcher, a list of strings the argument vector is prefixed with before the action is executed. The default `["env", "--"]` simply resolves the program to be executed in the current value of `PATH`, but a different value for the launcher can obviously be used to set environment variables like `LD_PRELOAD`. Relevant libraries and tools include [libfaketime](https://github.com/wolfcw/libfaketime), [fakehostname](https://github.com/dtcooper/fakehostname), and [disorderfs](https://salsa.debian.org/reproducible-builds/disorderfs). More variation can be achieved by comparing remote execution builds, either for two different remote-execution end points or comparing one remote-execution end point to the local build. The latter is also a good way to find out where a build that "works on my machine" differs. The endpoint on which the rebuild is executed can be set, in the same way as for build with the `-r` option; the cache end point to compare against can be set via the `--vs` option. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/target-file-glob-tree.md000066400000000000000000000426771516554100600266520ustar00rootroot00000000000000Target versus `FILE`, `GLOB`, and `TREE` ======================================== So far, we referred to defined targets as well as source files by their name and it just worked. When considering third-party software we already saw the `TREE` reference. In this section, we will highlight in more detail the ways to refer to sources, as well as the difference between defined and source targets. The latter is used, e.g., when third-party software has to be patched. As example for this section we use [gnu units](https://www.gnu.org/software/units/) where we want to patch into the standard units definition add two units of area popular in German news. Repository Config for `units` with patches ------------------------------------------ Before we begin, we first need to declare where the root of our workspace is located by creating the empty file `ROOT`: ``` sh $ touch ROOT ``` The sources are an archive available on the web. As upstream uses a different build system, we have to provide our own build description; we take the top-level directory as layer for this. As we also want to patch the definition file, we add the subdirectory `files` as logical repository for the patches. Hence we create a file `repos.json` with the following content. ``` {.jsonc srcname="repos.json"} { "main": "units" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } } , "import targets": {"repository": {"type": "file", "path": "."}} , "patches": {"repository": {"type": "file", "path": "files"}} , "units": { "repository": { "type": "archive" , "content": "9781174d42bd593d3bab6c6decfdcae60e3ce328" , "fetch": "https://ftp.gnu.org/gnu/units/units-2.21.tar.gz" , "subdir": "units-2.21" } , "target_root": "import targets" , "target_file_name": "TARGETS.units" , "bindings": {"rules": "rules-cc", "patches": "patches"} } } } ``` Patching a file: targets versus `FILE` -------------------------------------- Let's start by patching the source file `definitions.units`. While, conceptionally, we want to patch a third-party source file, we do *not* modify the sources. The workspace root is a git tree and stay like this. Instead, we remember that we specify *targets* and the definition of a target is looked up in the targets file; only if not defined there, it is implicitly considered a source target and taken from the target root. So we will define a *target* named `definitions.units` to replace the original source file. Let's first generate the patch. As we're already referring to source files as targets, we have to provide a targets file already; we start with the empty object and refine it later. ``` sh $ echo {} > TARGETS.units $ just-mr install -o . definitions.units INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","install","-C","...","-o",".","definitions.units"] INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Analysed target [["@","units","","definitions.units"],{}] INFO: Discovered 0 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","units","","definitions.units"],{}]. INFO: Processed 0 actions, 0 cache hits. INFO: Artifacts can be found in: /tmp/work/./definitions.units [0f24a321694aab5c1d3676e22d01fc73492bee42:342718:f] $ cp definitions.units definitions.units.orig $ # interactively edit definitions.units $ echo -e "/German units\n+2a\narea_soccerfield 105 m * 68 m\narea_saarland 2570 km^2\n.\nw\nq" | ed definitions.units 342718 # A few German units as currently in use. 342772 $ mkdir -p files $ echo {} > files/TARGETS $ diff -u definitions.units.orig definitions.units > files/definitions.units.diff $ rm definitions.units* ``` Our rules conveniently contain a rule `["patch", "file"]` to patch a single file, and we already created the patch. The only other input missing is the source file. So far, we could refer to it as `"definitions.units"` because there was no target of that name, but now we're about to define a target with that very name. Fortunately, in target files, we can use a special syntax to explicitly refer to a source file of the current module, even if there is a target with the same name: `["FILE", null, "definition.units"]`. The syntax requires the explicit `null` value for the current module, despite the fact that explicit file references are only allowed for the current module; in this way, the name is a list of length more than two and cannot be confused with a top-level module called `FILE`. So we add this target and obtain as `TARGETS.units` the following. ``` {.jsonc srcname="TARGETS.units"} { "definitions.units": { "type": ["@", "rules", "patch", "file"] , "src": [["FILE", ".", "definitions.units"]] , "patch": [["@", "patches", "", "definitions.units.diff"]] } } ``` Analysing `"definitions.units"` we find our defined target which contains an action output. Still, it looks like a patched source file; the new artifact is staged to the original location. Staging is also used in the action definition, to avoid magic names (like file names starting with `-`), in-place operations (all actions must not modify their inputs) and, in fact, have a fixed command line. ``` sh $ just-mr analyse definitions.units --dump-actions - INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","definitions.units","--dump-actions","-"] INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Result of target [["@","units","","definitions.units"],{}]: { "artifacts": { "definitions.units": {"data":{"id":"25d6d534845e948a49064e394712e529b4f6bd915de19d5d42549dacfa60dc41","path":"patched"},"type":"ACTION"} }, "provides": { }, "runfiles": { "definitions.units": {"data":{"id":"25d6d534845e948a49064e394712e529b4f6bd915de19d5d42549dacfa60dc41","path":"patched"},"type":"ACTION"} } } INFO: Actions for target [["@","units","","definitions.units"],{}]: [ { "command": ["sh","./run_patch.sh"], "env": { "PATH": "/bin:/usr/bin" }, "input": { "orig": { "data": { "file_type": "f", "id": "0f24a321694aab5c1d3676e22d01fc73492bee42", "size": 342718 }, "type": "KNOWN" }, "patch": { "data": { "path": "definitions.units.diff", "repository": "patches" }, "type": "LOCAL" }, "run_patch.sh": { "data": { "file_type": "f", "id": "85786bc8f6aeac0db3be48f8ce336f906e1d78a0", "size": 93 }, "type": "KNOWN" } }, "output": ["patched"] } ] $ ``` Building `"definitions.units"` we find out that the patch applied correctly ``` sh $ just-mr build definitions.units -P definitions.units | grep -A 5 'German units' INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","build","-C","...","definitions.units","-P","definitions.units"] INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Analysed target [["@","units","","definitions.units"],{}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 1 blobs INFO: Building [["@","units","","definitions.units"],{}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: definitions.units [763f3289422c296057e142f61be190ee6bef049a:342772:f] # A few German units as currently in use. # area_soccerfield 105 m * 68 m area_saarland 2570 km^2 zentner 50 kg $ ``` Globbing source files: `"GLOB"` ------------------------------- Next, we collect all `.units` files. We could simply do this by enumerating them in a target. ``` {.jsonc srcname="TARGETS.units"} ... , "data-draft": { "type": "install", "deps": ["definitions.units", "currency.units"]} ... ``` In this way, we get the desired collection of one unmodified source file and the output of the patch action. ``` sh $ just-mr analyse data-draft INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","data-draft"] INFO: Requested target is [["@","units","","data-draft"],{}] INFO: Result of target [["@","units","","data-draft"],{}]: { "artifacts": { "currency.units": {"data":{"file_type":"f","id":"ac6da8afaac0f34e114e123e4ab3a41e59121b10","size":14707},"type":"KNOWN"}, "definitions.units": {"data":{"id":"25d6d534845e948a49064e394712e529b4f6bd915de19d5d42549dacfa60dc41","path":"patched"},"type":"ACTION"} }, "provides": { }, "runfiles": { "currency.units": {"data":{"file_type":"f","id":"ac6da8afaac0f34e114e123e4ab3a41e59121b10","size":14707},"type":"KNOWN"}, "definitions.units": {"data":{"id":"25d6d534845e948a49064e394712e529b4f6bd915de19d5d42549dacfa60dc41","path":"patched"},"type":"ACTION"} } } $ ``` The disadvantage, however, that we might miss newly added `.units` files if we update and upstream added new files. So we want all source files that have the respective ending. The corresponding source reference is `"GLOB"`. A glob expands to the *collection* of all *sources* that are *files* in the *top-level* directory of the current module and that match the given pattern. It is important to understand this in detail and the rational behind it. - First of all, the artifact (and runfiles) map has an entry for each file that matches. In particular, targets have the option to define individual actions for each file, like `["CC", "binary"]` does for the source files. This is different from `"TREE"` where the artifact map contains a single artifact that happens to be a directory. The tree behaviour is preferable when the internals of the directory only matter for the execution of actions and not for analysis; then there are less entries to carry around during analysis and action-key computation, and the whole directory is "reserved" for that tree avoid staging conflicts when latter adding entries there. - As a source reference, a glob expands to explicit source files; targets having the same name as a source file are not taken into account. In our example, `["GLOB", null, "*.units"]` therefore contains the unpatched source file `definitions.units`. In this way, we avoid any surprises in the expansion of a glob when a new source file is added with a name equal to an already existing target. - Only files are considered for matching the glob. Directories are ignored. - Matches are only considered at the top-level directory. In this way, only one directory has to be read during analysis; allowing deeper globs would require traversal of subdirectories requiring larger cost. While the explicit `"TREE"` reference allows recursive traversal, in the typical use case of the respective workspace root being a `git` root, it is actually cheap; we can look up the `git` tree identifier without traversing the tree. Such a quick look up would not be possible if matches had to be selected. So, `["GLOB", null, "*.units"]` expands to all the relevant source files; but we still want to keep the patching. Most rules, like `"install"`, disallow staging conflicts to avoid accidentally ignoring a file due to conflicting name. In our case, however, the dropping of the source file in favour of the patched one is deliberate. For this, there is the rule `["data", "overlay"]` taking the union of the artifacts of the specified targets, accepting conflicts and resolving them in a latest-wins fashion. Keep in mind, that our target fields are list, not sets. Looking at the definition of the rule, one finds that it is simply a `"map_union"`. Hence we refine our `"data-draft"` target into the target `"data"` reading ``` {.jsonc srcname="TARGETS.units"} ... , "data": { "type": ["@", "rules", "data", "overlay"] , "deps": [["GLOB", null, "*.units"], "definitions.units"] } ... ``` The result of the analysis on this target, of course, remains the same. Finishing the example: binaries from globbed sources ---------------------------------------------------- The source-code organisation of units is pretty simple. All source and header files are in the top-level directory. As the header files are not in a directory of their own, we can't use a tree, so we use a glob, which is fine for the private headers of a binary. For the source files, we have to have them individually anyway. So our first attempt of defining the binary is as follows. ``` {.jsonc srcname="TARGETS.units"} ... , "units-draft": { "type": ["@", "rules", "CC", "binary"] , "name": ["units"] , "private-ldflags": ["-lm"] , "pure C": ["YES"] , "srcs": [["GLOB", null, "*.c"]] , "private-hdrs": [["GLOB", null, "*.h"]] } ... ``` The result basically work and shows that we have 5 source files in total, giving 5 compile and one link action. ``` sh $ just-mr build units-draft INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","build","-C","...","units-draft"] INFO: Requested target is [["@","units","","units-draft"],{}] INFO: Analysed target [["@","units","","units-draft"],{}] INFO: Discovered 6 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","units","","units-draft"],{}]. INFO (action:a6adae74a8b50103a46700203f32e2286cec0a31ac41667b88d5b1ebe9b9860c): Stderr of command ["cc","-I","work","-isystem","include","-c","work/strfunc.c","-o","work/strfunc.o"] in environment {"PATH":"/bin:/usr/bin"} work/strfunc.c:109:8: warning: extra tokens at end of #endif directive [-Wendif-labels] 109 | #endif NO_STRSPN | ^~~~~~~~~ INFO: Processed 6 actions, 0 cache hits. INFO: Artifacts built, logical paths are: units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] $ ``` To keep the build clean, we want to get rid of the warning. Of course, we could simply set an appropriate compiler flag, but let's do things properly and patch away the underlying reason. To do so, we first create a patch. ``` sh $ just-mr install -o . strfunc.c INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","install","-C","...","-o",".","strfunc.c"] INFO: Requested target is [["@","units","","strfunc.c"],{}] INFO: Analysed target [["@","units","","strfunc.c"],{}] INFO: Discovered 0 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","units","","strfunc.c"],{}]. INFO: Processed 0 actions, 0 cache hits. INFO: Artifacts can be found in: /tmp/work/./strfunc.c [e2aab4b825fa2822ccf33746d467a4944212abb9:2201:f] $ cp strfunc.c strfunc.c.orig $ echo -e "109\ns|N|// N\nw\nq" | ed strfunc.c 2201 #endif NO_STRSPN #endif // NO_STRSPN 2204 $ diff strfunc.c.orig strfunc.c > files/strfunc.c.diff $ rm strfunc.c* $ ``` Then we amend our `"units"` target. ``` {.jsonc srcname="TARGETS.units"} ... , "units": { "type": ["@", "rules", "CC", "binary"] , "name": ["units"] , "private-ldflags": ["-lm"] , "pure C": ["YES"] , "srcs": ["patched srcs"] , "private-hdrs": [["GLOB", null, "*.h"]] } , "patched srcs": { "type": ["@", "rules", "data", "overlay"] , "deps": [["GLOB", null, "*.c"], "strfunc.c"] } , "strfunc.c": { "type": ["@", "rules", "patch", "file"] , "src": [["FILE", ".", "strfunc.c"]] , "patch": [["@", "patches", "", "strfunc.c.diff"]] } ... ``` Building the new target, 2 actions have to be executed: the patching, and the compiling of the patched source file. As the patched file still generates the same object file as the unpatched file (after all, we only wanted to get rid of a warning), the linking step can be taken from cache. ``` sh $ just-mr build units INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","build","-C","...","units"] INFO: Requested target is [["@","units","","units"],{}] INFO: Analysed target [["@","units","","units"],{}] INFO: Discovered 7 actions, 0 tree overlays, 1 trees, 1 blobs INFO: Building [["@","units","","units"],{}]. INFO: Processed 7 actions, 5 cache hits. INFO: Artifacts built, logical paths are: units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] $ ``` To finish the example, we also add a default target (using that, if no target is specified, `just` builds the lexicographically first target), staging artifacts according to the usual conventions. ``` {.jsonc srcname="TARGETS.units"} ... , "": {"type": "install", "dirs": [["units", "bin"], ["data", "share/units"]]} ... ``` Then things work as expected ``` sh $ just-mr install -o /tmp/testinstall INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","install","-C","...","-o","/tmp/testinstall"] INFO: Requested target is [["@","units","",""],{}] INFO: Analysed target [["@","units","",""],{}] INFO: Discovered 8 actions, 0 tree overlays, 1 trees, 1 blobs INFO: Building [["@","units","",""],{}]. INFO: Processed 8 actions, 8 cache hits. INFO: Artifacts can be found in: /tmp/testinstall/bin/units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] /tmp/testinstall/share/units/currency.units [ac6da8afaac0f34e114e123e4ab3a41e59121b10:14707:f] /tmp/testinstall/share/units/definitions.units [763f3289422c296057e142f61be190ee6bef049a:342772:f] $ /tmp/testinstall/bin/units 'area_saarland' 'area_soccerfield' * 359943.98 / 2.7782101e-06 $ ``` just-buildsystem-justbuild-b1fb5fa/doc/tutorial/tests.md000066400000000000000000000457701516554100600237300ustar00rootroot00000000000000Creating Tests ============== To run tests with *justbuild*, we do **not** have a dedicated `test` subcommand. Instead, we consider tests being a specific action that generates a test report. Consequently, we use the `build` subcommand to build the test report, and thereby run the test action. Test actions, however, are slightly different from normal actions in that we don't want the build of the test report to be aborted if a test action fails (but still, we want only successfully actions taken from cache). Rules defining targets containing such special actions have to identify themselves as *tainted* by specifying a string explaining why such special actions are justified; in our case, the string is `"test"`. Besides the implicit marking by using a tainted rule, those tainting strings can also be explicitly assigned by the user in the definition of a target, e.g., to mark test data. Any target has to be tainted with (at least) all the strings any of its dependencies is tainted with. In this way, it is ensured that no test target will end up in a production build. For the remainder of this section, we expect to have the project files available resulting from successfully completing the tutorial section on [*Building C++ Hello World*](./hello-world.md). We will demonstrate how to write a test binary for the `greet` library and a shell test for the `helloworld` binary. Creating a C++ test binary -------------------------- First, we will create a C++ test binary for testing the correct functionality of the `greet` library. Therefore, we need to provide a C++ source file that performs the actual testing and returns non-`0` on failure. For simplicity reasons, we do not use a testing framework for this tutorial and place the tests into the same logical repository as the main project; when using additional third-party code for tests (like a test framework) it is advisable to put tests in a separate logical repository so that third-party test code is only fetched when testing is requested. Let us place this code in subdirectory `tests` ``` sh mkdir -p ./tests ``` A simple test that captures standard output and verifies it with the expected output should be provided in the file `tests/greet.test.cpp`: ``` {.cpp srcname="tests/greet.test.cpp"} #include #include #include #include #include "greet/greet.hpp" template auto capture_stdout(std::function const& func) -> std::string { int fd[2]; if (pipe(fd) < 0) return {}; int fd_stdout = dup(fileno(stdout)); fflush(stdout); dup2(fd[1], fileno(stdout)); func(); fflush(stdout); std::string buf(kMaxBufSize, '\0'); auto n = read(fd[0], &(*buf.begin()), kMaxBufSize); close(fd[0]); close(fd[1]); dup2(fd_stdout, fileno(stdout)); return buf.substr(0, n); } auto test_greet(std::string const& name) -> bool { auto expect = std::string{"Hello "} + name + "!\n"; auto result = capture_stdout([&name] { greet(name); }); std::cout << "greet output: " << result << std::endl; return result == expect; } int main() { return test_greet("World") && test_greet("Universe") ? 0 : 1; } ``` Next, a new test target needs to be created in module `greet`. This target uses the rule `["@", "rules", "CC/test", "test"]` and needs to depend on the `["greet", "greet"]` target. To create the test target, add the following to `tests/TARGETS`: ``` {.jsonc srcname="tests/TARGETS"} { "greet": { "type": ["@", "rules", "CC/test", "test"] , "name": ["test_greet"] , "srcs": ["greet.test.cpp"] , "private-deps": [["greet", "greet"]] } } ``` Before we can run the test, a proper default module for `CC/test` must be provided. By specifying the appropriate target in this module the default test runner can be overwritten by a different test runner fom the rule's workspace root. Moreover, all test targets share runner infrastructure from `shell/test`, e.g., summarizing multiple runs per test (to detect flakiness) if the configuration variable `RUNS_PER_TEST` is set. The precise location of those implicit dependencies can be seen via `just describe`. ``` sh $ just-mr describe tests greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","describe","-C","...","tests","greet"] [["@","tutorial","tests","greet"],{}] is defined by user-defined rule ["@","rules-cc","CC/test","test"]. | A test written in C++ String fields - "name" | The name of the test ... - implicit dependency | The C/C++ toolchain to use - ["@","rules-cc","CC","defaults"] - implicit dependency | The test runner which starts the actual test binary after providing | the respective environment. The runner also takes care of capturing | stdout/stderr and timing information. - ["@","rules-cc","CC/test","runner"] - implicit dependency | The shell toolchain to use PATH from for calling the summary action - ["@","rules-cc","shell","defaults"] - implicit dependency | Tool to aggregate the results of individual test runs (for flakyness | detection) to an overall test result. If more fields than the result | itself is needed, those can be specified using the "summarizer" rule. - ["@","rules-cc","shell/test","summarizer"] ... ``` However, in our case, we want to use the default runner and therefore it is sufficient to create an empty module. To do so, we create subdirectories ``` sh $ mkdir -p ./tutorial-defaults/CC/test $ mkdir -p ./tutorial-defaults/shell/test ``` where we create, respectively, the file `tutorial-defaults/CC/test/TARGETS` with content ``` {.jsonc srcname="tutorial-defaults/CC/test/TARGETS"} {} ``` as well as the file `tutorial-defaults/shell/test/TARGETS` with content ``` {.jsonc srcname="tutorial-defaults/shell/test/TARGETS"} {} ``` and the file `tutorial-defulats/shell` with content ``` {.jsonc srcname="tutorial-defaults/shell/TARGETS"} {"defaults": {"type": "defaults"}} ``` indicating that we just use defaults for the shell defaults. Now we can run the test (i.e., build the test result): ``` sh $ just-mr build tests greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","tests","greet"] INFO: Requested target is [["@","tutorial","tests","greet"],{}] INFO: Analysed target [["@","tutorial","tests","greet"],{}] INFO: Target tainted ["test"]. INFO: Discovered 5 actions, 0 tree overlays, 3 trees, 1 blobs INFO: Building [["@","tutorial","tests","greet"],{}]. INFO: Processed 5 actions, 2 cache hits. INFO: Artifacts built, logical paths are: pwd [22b121af0c81943f8174754055d169f69d95789c:313:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] stderr [8b137891791fe96927ad78e64b0aad7bded08bdc:1:f] stdout [ae6c6813755da67954a4a562f6d2ef01578c3e89:60:f] time-start [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] time-stop [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] (1 runfiles omitted.) INFO: Target tainted ["test"]. $ ``` Note that the target is correctly reported as tainted with `"test"`. It will produce 3 additional actions for compiling, linking and running the test binary. The result of the test target is formed of 5 artifacts: `result` (containing `UNKNOWN`, `PASS`, or `FAIL`), `stderr`, `stdout`, `time-start`, and `time-stop`, and a single runfile (omitted in the output above), which is a tree artifact with the name `test_greet` that contains all of the above artifacts. The test was run successfully as otherwise all reported artifacts would have been reported as `FAILED` in the output, and *justbuild* would have returned the exit code `2`. To immediately print the standard output produced by the test binary on the command line, the `-P` option can be used. Argument to this option is the name of the artifact that should be printed on the command line, in our case `stdout`: ``` sh $ just-mr build -P stdout tests greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-P","stdout","tests","greet"] INFO: Requested target is [["@","tutorial","tests","greet"],{}] INFO: Analysed target [["@","tutorial","tests","greet"],{}] INFO: Target tainted ["test"]. INFO: Discovered 5 actions, 0 tree overlays, 3 trees, 1 blobs INFO: Building [["@","tutorial","tests","greet"],{}]. INFO: Processed 5 actions, 5 cache hits. INFO: Artifacts built, logical paths are: pwd [22b121af0c81943f8174754055d169f69d95789c:313:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] stderr [8b137891791fe96927ad78e64b0aad7bded08bdc:1:f] stdout [ae6c6813755da67954a4a562f6d2ef01578c3e89:60:f] time-start [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] time-stop [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] (1 runfiles omitted.) greet output: Hello World! greet output: Hello Universe! INFO: Target tainted ["test"]. $ ``` To also strip *justbuild*'s `INFO:` prints from this output, the argument `--log-limit 1` can be passed to the `just-mr` call. Our test binary does not have any useful options for directly interact with it. When working with test frameworks, it sometimes can be desirable to get hold of the test binary itself for manual interaction. The running of the test binary is the last action associated with the test and the test binary is, of course, one of its inputs. ``` sh $ just-mr analyse --request-action-input -1 tests greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","analyse","-C","...","--request-action-input","-1","tests","greet"] INFO: Requested target is [["@","tutorial","tests","greet"],{}] INFO: Request is input of action #-1 INFO: Analysed target [["@","tutorial","tests","greet"],{}] INFO: Result of input of action #-1 of target [["@","tutorial","tests","greet"],{}]: { "artifacts": { "runner": {"data":{"file_type":"x","id":"4984b1766a38849c7039f8ae9ede9dae891eebc3","size":2004},"type":"KNOWN"}, "test": {"data":{"id":"cd8801144beca99330c0003ea464f7c44e9547f4582317f5037b304425ee2d8b","path":"work/test_greet"},"type":"ACTION"}, "test-args.json": {"data":{"file_type":"f","id":"0637a088a01e8ddab3bf3fa98dbe804cbde1a0dc","size":2},"type":"KNOWN"}, "test-launcher.json": {"data":{"file_type":"f","id":"0637a088a01e8ddab3bf3fa98dbe804cbde1a0dc","size":2},"type":"KNOWN"} }, "provides": { "cmd": [ "./runner" ], "env": { }, "may_fail": "CC test /test_greet failed", "output": [ "pwd", "result", "stderr", "stdout", "time-start", "time-stop" ], "output_dirs": [ ] }, "runfiles": { } } INFO: Target tainted ["test"]. $ ``` The provided data also shows us the precise description of the action for which we request the input. This allows us to manually rerun the action. Or we can simply interact with the test binary manually after installing the inputs to this action. Requesting the inputs of an action can also be useful when debugging a build failure. ``` sh $ just-mr install -o work --request-action-input -1 tests greet INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","install","-C","...","-o","work","--request-action-input","-1","tests","greet"] INFO: Requested target is [["@","tutorial","tests","greet"],{}] INFO: Request is input of action #-1 INFO: Analysed target [["@","tutorial","tests","greet"],{}] INFO: Target tainted ["test"]. INFO: Discovered 5 actions, 0 tree overlays, 3 trees, 1 blobs INFO: Building input of action #-1 of [["@","tutorial","tests","greet"],{}]. INFO: Processed 4 actions, 4 cache hits. INFO: Artifacts can be found in: /tmp/tutorial/work/runner [4984b1766a38849c7039f8ae9ede9dae891eebc3:2004:x] /tmp/tutorial/work/test [306e9440ba06bf615f51d84cde0ce76563723c3d:24448:x] /tmp/tutorial/work/test-args.json [0637a088a01e8ddab3bf3fa98dbe804cbde1a0dc:2:f] /tmp/tutorial/work/test-launcher.json [0637a088a01e8ddab3bf3fa98dbe804cbde1a0dc:2:f] INFO: Target tainted ["test"]. $ cd work/ $ ./test greet output: Hello World! greet output: Hello Universe! $ echo $? 0 $ cd .. $ rm -rf work ``` When looking at the reported actions, we also see the difference between the action graph and the part of the action graph that is needed to compute the requested artifacts. Targets are always analyzed completely, including all actions occurring in their definition. When building, however, only that part of the graph is traversed that is needed for the requested artifacts. In our case, the actual test action is not considered in the build, even though it is part of the definition of the target. A larger difference between actions discovered in the analysis and actions processed during the build can occur when rules only use parts of a target; consider, e.g., the auxiliary target `just-ext-hdrs` that collects the (partially generated) header files of the external dependencies, but not the actual libraries. In this case, the actions for generating those libraries (compiling sources, calling the archive tool) are discovered when analyzing the target, but never visited during the build. Creating a shell test --------------------- Similarly, to create a shell test for testing the `helloworld` binary, a test script `tests/test_helloworld.sh` must be provided: ``` {.sh srcname="tests/test_helloworld.sh"} set -e [ "$(./helloworld)" = "Hello Universe!" ] ``` The test target for this shell tests uses the rule `["@", "rules", "shell/test", "script"]` and must depend on the `"helloworld"` target. To create the test target, add the following to the `tests/TARGETS` file: ``` {.jsonc srcname="tests/TARGETS"} ... , "helloworld": { "type": ["@", "rules", "shell/test", "script"] , "name": ["test_helloworld"] , "test": ["test_helloworld.sh"] , "deps": [["", "helloworld"]] } ... ``` A shell test depends on the default settings for the shell. Therefore, if we bring our own toolchain defaults for our rules, we have to do this here as well. In this case, however, we have already created the toolchain description before running the C++ test, as that also uses the shell toolchain for the summarizing results. Now we can run the shell test (i.e., build the test result): ``` sh $ just-mr build tests helloworld INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","tests","helloworld"] INFO: Requested target is [["@","tutorial","tests","helloworld"],{}] INFO: Analysed target [["@","tutorial","tests","helloworld"],{}] INFO: Target tainted ["test"]. INFO: Discovered 5 actions, 0 tree overlays, 4 trees, 2 blobs INFO: Building [["@","tutorial","tests","helloworld"],{}]. INFO: Processed 5 actions, 4 cache hits. INFO: Artifacts built, logical paths are: pwd [00dc0e2b2c82d1cc616173b339e220249fe9debd:313:f] result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] stderr [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] stdout [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] time-start [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] time-stop [7ca67f9c9a4b4f2b1948cc769469f6476e7bd320:11:f] (1 runfiles omitted.) INFO: Target tainted ["test"]. $ ``` The result is also similar, containing also the 5 artifacts and a single runfile (omitted in the output above), which is a tree artifact with the name `test_helloworld` that contains all of the above artifacts. Creating a compound test target ------------------------------- As most people probably do not want to call every test target by hand, it is desirable to have a compound test target that triggers the build of multiple test reports. The most simple way to do so is to use an `"install"` target. The field `"deps"` of an install target is a list of targets for which the runfiles are collected. As for the tests the runfiles happen to be tree artifacts named the same way as the test and containing all test results, this is precisely what we need. Furthermore, as the dependent test targets are tainted by `"test"`, also the compound test target must be tainted by the same string. To create the compound test target combining the two tests above (the tests `"greet"` and `"helloworld"` from module `"tests"`), add the following to the `tests/TARGETS` file: ``` {.jsonc srcname="tests/TARGETS"} ... , "ALL-simple": { "type": "install" , "tainted": ["test"] , "deps": ["greet", "helloworld"] } ... ``` Now we can run all tests at once by just building the compound test target `"ALL-simple"`: ``` sh $ just-mr build tests ALL-simple INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","tests","ALL-simple"] INFO: Requested target is [["@","tutorial","tests","ALL-simple"],{}] INFO: Analysed target [["@","tutorial","tests","ALL-simple"],{}] INFO: Target tainted ["test"]. INFO: Discovered 8 actions, 0 tree overlays, 5 trees, 3 blobs INFO: Building [["@","tutorial","tests","ALL-simple"],{}]. INFO: Processed 8 actions, 8 cache hits. INFO: Artifacts built, logical paths are: test_greet [37b6cb11b5bf862f370df78ef3c0ae9b1437ffeb:208:t] test_helloworld [1345b72ef6ec592fd83ac0614343ab9042358bcd:208:t] INFO: Target tainted ["test"]. $ ``` As a result it reports the runfiles (result directories) of both tests as artifacts. Both tests ran successfully as none of those artifacts in this output above are tagged as `FAILED`. While the built-in `"install"` rule works in principle, the preferred way is to use the rule `["@", "rules", "test", "suite"]`. For artifacts and runfiles it does the same as `"install"`, however it also propagates the provided information appropriately; this means, it can also be used, e.g., as a target for [linting](lint.md) allowing in a single target to lint all code that went into all tests. The usage is similar, and, as a test target, it is also implicitly tainted. ``` {.jsonc srcname="tests/TARGETS"} ... , "ALL": { "type": ["@", "rules", "test", "suite"] , "deps": ["greet", "helloworld"] } ... ``` Again, we can run all tests at once by building the compound target. ``` sh $ just-mr build tests ALL INFO: Performing repositories setup INFO: Found 3 repositories involved INFO: Setup finished, exec ["just","build","-C","...","tests","ALL"] INFO: Requested target is [["@","tutorial","tests","ALL"],{}] INFO: Analysed target [["@","tutorial","tests","ALL"],{}] INFO: Target tainted ["test"]. INFO: Discovered 8 actions, 0 tree overlays, 5 trees, 3 blobs INFO: Building [["@","tutorial","tests","ALL"],{}]. INFO: Processed 8 actions, 8 cache hits. INFO: Artifacts built, logical paths are: test_greet [37b6cb11b5bf862f370df78ef3c0ae9b1437ffeb:208:t] test_helloworld [1345b72ef6ec592fd83ac0614343ab9042358bcd:208:t] INFO: Target tainted ["test"]. $ ``` As the output structure is again one tree per test, test suits can be put into other test suites; to avoid conflicts, the `"stage"` field of the test suite can be used to put all outputs into a suite-specific subdirectory. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/third-party-software.md000066400000000000000000000444631516554100600266630ustar00rootroot00000000000000Building Third-party Software ============================= Third-party projects usually ship with their own build description, which often happens to not be compatible with *justbuild*. Nevertheless, it is often desireable to include external projects via their source code base, instead of relying on the integration of out-of-band binary distributions. *justbuild* offers a flexible approach to provide the required build description via an overlay layer without the need to touch the original code base. This mechanism is independent of the actual *justbuild* description eventually used, and the latter might well be a [rule calling a foreign buildsystem](https://github.com/just-buildsystem/rules-cc#rule-ccforeigncmake-library). In this section, however, we describe the cleaner approach of providing a native build description. For the remainder of this section, we expect to have the project files available resulting from successfully completing the tutorial section on [*Building C++ Hello World*](./hello-world.md). We will demonstrate how to use the open-source project [fmtlib](https://github.com/fmtlib/fmt) as an example for integrating third-party software to a *justbuild* project. Creating the target overlay layer for fmtlib -------------------------------------------- Before we construct the overlay layer for fmtlib, we need to determine its file structure ([tag 8.1.1](https://github.com/fmtlib/fmt/tree/8.1.1)). The relevant header and source files are structured as follows: fmt | +--include | +--fmt | +--*.h | +--src +--format.cc +--os.cc The public headers can be found in `include/fmt`, while the library's source files are located in `src`. For the overlay layer, the `TARGETS` files should be placed in a tree structure that resembles the original code base's structure. It is also good practice to provide a top-level `TARGETS` file, leading to the following structure for the overlay: fmt-layer | +--TARGETS +--include | +--fmt | +--TARGETS | +--src +--TARGETS Let's create the overlay structure: ``` sh $ mkdir -p ./fmt-layer/include/fmt $ mkdir -p ./fmt-layer/src ``` The directory `include/fmt` contains only header files. As we want all files in this directory to be included in the `"hdrs"` target, we can safely use the explicit `TREE` reference[^1], which collects, in a single artifact (describing a directory) *all* directory contents from `"."` of the workspace root. Note that the `TARGETS` file is only part of the overlay, and therefore will not be part of this tree. Furthermore, this tree should be staged to `"fmt"`, so that any consumer can include those headers via ``. The resulting header directory target `"hdrs"` in `include/fmt/TARGETS` should be described as: ``` {.jsonc srcname="fmt-layer/include/fmt/TARGETS"} { "hdrs": { "type": ["@", "rules", "data", "staged"] , "srcs": [["TREE", null, "."]] , "stage": ["fmt"] } } ``` The actual library target is defined in the directory `src`. For the public headers, it refers to the previously created `"hdrs"` target via its fully-qualified target name (`["include/fmt", "hdrs"]`). Source files are the two local files `format.cc`, and `os.cc`. The final target description in `src/TARGETS` will look like this: ``` {.jsonc srcname="fmt-layer/src/TARGETS"} { "fmt": { "type": ["@", "rules", "CC", "library"] , "name": ["fmt"] , "hdrs": [["include/fmt", "hdrs"]] , "srcs": ["format.cc", "os.cc"] } } ``` Finally, the top-level `TARGETS` file can be created. While it is technically not strictly required, it is considered good practice to *export* every target that may be used by another project. Exported targets are subject to high-level target caching, which allows to skip the analysis and traversal of entire subgraphs in the action graph. Therefore, we create an export target that exports the target `["src", "fmt"]`, with only the variables in the field `"flexible_config"` being propagated. The top-level `TARGETS` file contains the following content: ``` {.jsonc srcname="fmt-layer/TARGETS"} { "fmt": { "type": "export" , "target": ["src", "fmt"] , "flexible_config": [ "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "DWP" , "ENV" , "DEBUG" ] } } ``` After adding the library to the multi-repository configuration (next step), the list of configuration variables a target, like `["src", "fmt"]`, actually depends on can be obtained using the `--dump-vars` option of the `analyse` subcommand. In this way, an informed decision can be taken when deciding which variables of the export target to make tunable for the consumer. Adding fmtlib to the Multi-Repository Configuration --------------------------------------------------- Based on the *hello world* tutorial, we can extend the existing `repos.json` by the layer definition `"fmt-targets-layer"` and the repository `"fmtlib"`, which is based on the Git repository with its target root being overlayed. Furthermore, we want to use `"fmtlib"` in the repository `"tutorial"`, and therefore need to introduce an additional binding `"format"` for it: ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": {"type": "file", "path": "./tutorial-defaults"} } , "fmt-targets-layer": { "repository": {"type": "file", "path": "./fmt-layer"} } , "fmtlib": { "repository": { "type": "git" , "branch": "8.1.1" , "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9" , "repository": "https://github.com/fmtlib/fmt.git" } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } } ``` This `"format"` binding can be used to add a new private dependency in `greet/TARGETS`: ``` {.jsonc srcname="greet/TARGETS"} { "greet": { "type": ["@", "rules", "CC", "library"] , "name": ["greet"] , "hdrs": ["greet.hpp"] , "srcs": ["greet.cpp"] , "stage": ["greet"] , "private-deps": [["@", "format", "", "fmt"]] } } ``` Consequently, the `fmtlib` library can now be used by `greet/greet.cpp`: ``` {.cpp srcname="greet/greet.cpp"} #include "greet.hpp" #include void greet(std::string const& s) { fmt::print("Hello {}!\n", s); } ``` Due to changes made to `repos.json`, building this tutorial requires to rerun `just-mr`, which will fetch the necessary sources for the external repositories: ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 0 cached, 0 uncached, 1 not eligible for caching INFO: Discovered 7 actions, 0 tree overlays, 3 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 7 actions, 1 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` Note that in order to build the `fmt` target alone, its containing repository `fmtlib` must be specified via the `--main` option: ``` sh $ just-mr --main fmtlib build fmt INFO: Performing repositories setup INFO: Found 4 repositories involved INFO: Setup finished, exec ["just","build","-C","...","fmt"] INFO: Requested target is [["@","fmtlib","","fmt"],{}] INFO: Analysed target [["@","fmtlib","","fmt"],{}] INFO: Export targets found: 0 cached, 0 uncached, 1 not eligible for caching INFO: Discovered 3 actions, 0 tree overlays, 1 trees, 0 blobs INFO: Building [["@","fmtlib","","fmt"],{}]. INFO: Processed 3 actions, 3 cache hits. INFO: Artifacts built, logical paths are: libfmt.a [513b2ac17c557675fc841f3ebf279003ff5a73ae:240914:f] (1 runfiles omitted.) $ ``` Employing high-level target caching ----------------------------------- To make use of high-level target caching for exported targets, we need to ensure that all inputs to an export target are transitively content-fixed. This is automatically the case for `"type":"git"` repositories. However, the `libfmt` repository also depends on `"rules-cc"`, `"tutorial-defaults"`, and `"fmt-target-layer"`. As the latter two are `"type":"file"` repositories, they must be put under Git versioning first: ``` sh $ git init . $ git add tutorial-defaults fmt-layer $ git commit -m "fix compile flags and fmt targets layer" [master (root-commit) 0337c65] fix compile flags and fmt targets layer 4 files changed, 37 insertions(+) create mode 100644 fmt-layer/TARGETS create mode 100644 fmt-layer/include/fmt/TARGETS create mode 100644 fmt-layer/src/TARGETS create mode 100644 tutorial-defaults/CC/TARGETS ``` Note that `rules-cc` already is under Git versioning. Now, to instruct `just-mr` to use the content-fixed, committed source trees of those `"type":"file"` repositories the pragma `"to_git"` must be set for them in `repos.json`: ``` {.jsonc srcname="repos.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "fmt-targets-layer": { "repository": { "type": "file" , "path": "./fmt-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": { "type": "git" , "branch": "8.1.1" , "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9" , "repository": "https://github.com/fmtlib/fmt.git" } , "target_root": "fmt-targets-layer" , "bindings": {"rules": "rules-cc"} } } } ``` Due to changes in the repository configuration, we need to rebuild and the benefits of the target cache should be visible on the second build: ``` sh $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 0 cached, 1 uncached, 0 not eligible for caching INFO: Discovered 7 actions, 0 tree overlays, 3 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 7 actions, 7 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] INFO: Backing up artifacts of 1 export targets $ $ just-mr build helloworld INFO: Performing repositories setup INFO: Found 5 repositories involved INFO: Setup finished, exec ["just","build","-C","...","helloworld"] INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] INFO: Export targets found: 1 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 0 tree overlays, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 4 cache hits. INFO: Artifacts built, logical paths are: helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` Note that in the second run the export target `"fmt"` was taken from cache and its 3 actions were eliminated, as their result has been recorded to the high-level target cache during the first run. Also note the final message in the first run. As that was the first time the export target `"fmt"` was built (i.e., target `"fmt"` with default configuration flags), an entry in the target-level cache was created. The log message showcases that when a remote-execution endpoint is involved, any artifacts referenced by a built export target needs to be ensured to be available. Combining overlay layers for multiple projects ---------------------------------------------- Projects typically depend on multiple external repositories. Creating an overlay layer for each external repository might unnecessarily clutter up the repository configuration and the file structure of your repository. One solution to mitigate this issue is to combine the `TARGETS` files of multiple external repositories in a single overlay layer. To avoid conflicts, the `TARGETS` files can be assigned different file names per repository. As an example, imagine a common overlay layer with the files `TARGETS.fmt` and `TARGETS.gsl` for the repositories `"fmtlib"` and `"gsl-lite"`, respectively: common-layer | +--TARGETS.fmt +--TARGETS.gsl +--include | +--fmt | | +--TARGETS.fmt | +--gsl | +--TARGETS.gsl | +--src +--TARGETS.fmt Such a common overlay layer can be used as the target root for both repositories with only one difference: the `"target_file_name"` field. By specifying this field, the dispatch where to find the respective target description for each repository is implemented. For the given example, the following `repos.json` defines the overlay `"common-targets-layer"`, which is used by `"fmtlib"` and `"gsl-lite"`: ``` {.jsonc srcname="repos.gsl-lite.json"} { "main": "tutorial" , "repositories": { "rules-cc": { "repository": { "type": "git" , "branch": "master" , "commit": "7a2fb9f639a61cf7b7d7e45c7c4cea845e7528c6" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } , "target_root": "tutorial-defaults" , "rule_root": "rules-cc" } , "tutorial": { "repository": {"type": "file", "path": "."} , "bindings": {"rules": "rules-cc", "format": "fmtlib"} } , "tutorial-defaults": { "repository": { "type": "file" , "path": "./tutorial-defaults" , "pragma": {"to_git": true} } } , "common-targets-layer": { "repository": { "type": "file" , "path": "./common-layer" , "pragma": {"to_git": true} } } , "fmtlib": { "repository": { "type": "git" , "branch": "8.1.1" , "commit": "b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9" , "repository": "https://github.com/fmtlib/fmt.git" } , "target_root": "common-targets-layer" , "target_file_name": "TARGETS.fmt" , "bindings": {"rules": "rules-cc"} } , "gsl-lite": { "repository": { "type": "git" , "branch": "v0.40.0" , "commit": "d6c8af99a1d95b3db36f26b4f22dc3bad89952de" , "repository": "https://github.com/gsl-lite/gsl-lite.git" } , "target_root": "common-targets-layer" , "target_file_name": "TARGETS.gsl" , "bindings": {"rules": "rules-cc"} } } } ``` Using pre-built dependencies ---------------------------- While building external dependencies from source brings advantages, most prominently the flexibility to quickly and seamlessly switch to a different build configuration (production, debug, instrumented for performance analysis; cross-compiling for a different target architecture), there are also legitimate reasons to use pre-built dependencies. The most prominent one is if your project is packaged as part of a larger distribution. For that reason, just also has target files for all its dependencies assuming they are pre-installed. The reason why target files are used at all for this situation is twofold. - On the one hand, having a target allows the remaining targets to not care about where their dependencies come from, or if it is a build against pre-installed dependencies or not. Also, the top-level binary does not have to know the linking requirements of its transitive dependencies. In other words, information stays where it belongs to and if one target acquires a new dependency, the information is automatically propagated to all targets using it. - Still some information is needed to use a pre-installed library and, as explained, a target describing the pre-installed library is the right place to collect this information. - The public header files of the library. By having this explicit, we do not accumulate directories in the include search path and hence also properly detect include conflicts. - The information on how to link the library itself (i.e., basically its base name). - Any dependencies on other libraries that the library might have. This information is used to obtain the correct linking order and complete transitive linking arguments while keeping the description maintainable, as each target still only declares its direct dependencies. A target description for a pre-built version of the format library that was used as an example in this section is shown next; with our staging mechanism the logical repository it belongs to is rooted in the `fmt` subdirectory of the `include` directory of the ambient system. ``` {.jsonc srcname="etc/import.prebuilt/TARGETS.fmt"} { "fmt": { "type": ["@", "rules", "CC", "library"] , "name": ["fmt"] , "stage": ["fmt"] , "hdrs": [["TREE", null, "."]] , "private-ldflags": ["-lfmt"] } } ``` However, even specifying all the include locations and headers can be tedious and, in the end, it is information that `pkg-config` can provide as well. So there is a rule to import libraries that way and the actual packaging-build version of `libfmt`, as provided in `etc/import.pkgconfig`, looks as follows. ``` {.jsonc srcname="etc/import.pkgconfig/TARGETS.fmt"} { "fmt": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["fmt"]} } ``` [^1]: Explicit `TREE` references are always a list of length 3, to distinguish them from target references of length 2 (module and target name). Furthermore, the second list element is always `null` as we only want to allow tree references from the current module. just-buildsystem-justbuild-b1fb5fa/doc/tutorial/tree-overlay.md000066400000000000000000000177361516554100600252050ustar00rootroot00000000000000# Tree Overlays The underlying idea of a tree object is that it is an opaque object, that can be passed around as a single artifact. Trees can be obtained as a directory output of an action or as explicit reference to a source directory. Using a tree rather than a collection of individual files can be useful for reserving a whole directory for headers for a particular library in order to avoid future conflicts, or if the individual outputs of an action are not known in advance. The latter happens often when calling foreign build systems to build a third-party library or even toolchain; still, usually those outputs can be used as opaque trees using appropriate staging. There are, however, a few examples where opaque trees have to be combined into a single one, e.g., if there is an external requirement that certain files be staged flatly in a single directory. To also support those rare use cases, `just` supports in-memory actions to compute the overlay of two trees, optionally rejecting conflicts instead of resolving them in a latest-wins way. Those actions can be requested in user-defined rules, as well as by the built-in rules `tree_overlay` and `disjoint_tree_overlay` (where the latter causes a build error if the trees cannot be overlayed in a conflict-free way). Here we demonstrate the way these rules work on an artificial example which we start from scratch. ``` sh $ touch ROOT ``` We simply work locally, so our `repos.json` is trivial. ``` {.jsonc srcname="repos.json"} {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} ``` As a replacement for a foreign-build-system call, consider a target having a single tree artifact as output, with contents heavily depending on the configuration. ``` {.jsonc srcname="TARGETS"} { "foo": { "type": "generic" , "arguments_config": ["FOO_BINS"] , "out_dirs": ["bin"] , "cmds": [ "mkdir -p bin" , { "type": "join" , "$1": [ "for tool in " , {"type": "join_cmd", "$1": {"type": "var", "name": "FOO_BINS"}} , " ; do echo \"foo binary ${tool}\" > bin/\"${tool}\"" , " ; chmod 755 bin/\"${tool}\"" , " ; done" ] } ] } } ``` So, depending on the configuration the output tree has different entries. ``` sh $ just-mr build -D '{"FOO_BINS": ["version", "upload", "download"]}' -p foo INFO: Performing repositories setup INFO: Found 1 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"FOO_BINS\": [\"version\", \"upload\", \"download\"]}","-p"] INFO: Requested target is [["@","","","foo"],{"FOO_BINS":["version","upload","download"]}] INFO: Analysed target [["@","","","foo"],{"FOO_BINS":["version","upload","download"]}] INFO: Discovered 1 actions, 0 tree overlays, 0 trees, 0 blobs INFO: Building [["@","","","foo"],{"FOO_BINS":["version","upload","download"]}]. INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: bin [8cd3ecc03f0ba26d9e104e52b40f88a0bc5a84b9:105:t] { "download": "[f270834a3411ba7d9e6fab59a8d93e1fbd6e55a3::x]", "upload": "[af38d5c40e8828c67d0031fce42da86b68f56182::x]", "version": "[a263fff2a3b94429878303875861fd93bcdbe248::x]" } $ just-mr build -D '{"FOO_BINS": ["version", "ci", "co", "rlog"]}' -p foo ... INFO: Processed 1 actions, 0 cache hits. INFO: Artifacts built, logical paths are: bin [eaee8451fe929904016b73c73f466f534e248c3d:127:t] { "ci": "[eb8e3ec5baca0f16da7fe8b200181bde123c11bf::x]", "co": "[1cb77df2638381460b338882e2942b3eb0a09975::x]", "rlog": "[157a0fca577e8717d68920f76a998ab1b398594d::x]", "version": "[a263fff2a3b94429878303875861fd93bcdbe248::x]" } ``` Now, assume we have another such target. ``` {.jsonc srcname="TARGETS"} ... , "bar": { "type": "generic" , "arguments_config": ["BAR_BINS"] , "out_dirs": ["bin"] , "cmds": [ "mkdir -p bin" , { "type": "join" , "$1": [ "for tool in " , {"type": "join_cmd", "$1": {"type": "var", "name": "BAR_BINS"}} , " ; do echo \"bar binary ${tool}\" > bin/\"${tool}\"" , " ; chmod 755 bin/\"${tool}\"" , " ; done" ] } ] } ... ``` Both targets produce a tree `bin` instead of a collection of files within the directory `bin`. Therefore, an overlay at analysis time could only result in one or the other directory. However, we overlay the results at build time. ``` {.jsonc srcname="TARGETS"} ... , "both": {"type": "tree_overlay", "deps": ["foo", "bar"]} , "both-noconflict": {"type": "disjoint_tree_overlay", "deps": ["foo", "bar"]} ... ``` If the entries do not conflict, in both cases, we get the union of the files. ``` sh $ just-mr build -D '{"FOO_BINS": ["ci", "co"], "BAR_BINS": ["up", "down"]}' -P bin both ... INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: [98e5f0eed05bf887a6c9df7616b7d83323acf677:30:t] INFO: 'bin' not a direct logical path of the specified target; will take subobject 'bin' of '' { "ci": "[eb8e3ec5baca0f16da7fe8b200181bde123c11bf::x]", "co": "[1cb77df2638381460b338882e2942b3eb0a09975::x]", "down": "[53aa524d39aff972c976bfd729a3fd26c5d364fd::x]", "up": "[3bfadc230e82da61f056e1d9acd854298f0b19c3::x]" } $ just-mr build -D '{"FOO_BINS": ["ci", "co"], "BAR_BINS": ["up", "down"]}' -P bin both-noconflict ... INFO: Processed 2 actions, 2 cache hits. INFO: Artifacts built, logical paths are: [98e5f0eed05bf887a6c9df7616b7d83323acf677:30:t] INFO: 'bin' not a direct logical path of the specified target; will take subobject 'bin' of '' { "ci": "[eb8e3ec5baca0f16da7fe8b200181bde123c11bf::x]", "co": "[1cb77df2638381460b338882e2942b3eb0a09975::x]", "down": "[53aa524d39aff972c976bfd729a3fd26c5d364fd::x]", "up": "[3bfadc230e82da61f056e1d9acd854298f0b19c3::x]" } ``` In case of a conflict, however, the targets differ. The first one, `both`, will overlay the files in a latest-wins fashion, whereas the second will fail when handling the overlay action. ``` sh $ just-mr build -D '{"FOO_BINS": ["version", "ci", "co"], "BAR_BINS": ["version", "up", "down"]}' -P bin/version both ... INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: [2a26b3dc1774df55eb1f1d9a865611a413204ab2:30:t] INFO: 'bin/version' not a direct logical path of the specified target; will take subobject 'bin/version' of '' bar binary version $ just-mr build -D '{"FOO_BINS": ["version", "ci", "co"], "BAR_BINS": ["version", "up", "down"]}' both-noconflict || : INFO: Performing repositories setup INFO: Found 1 repositories involved INFO: Setup finished, exec ["just","build","-C","...","-D","{\"FOO_BINS\": [\"version\", \"ci\", \"co\"], \"BAR_BINS\": [\"version\", \"up\", \"down\"]}","both-noconflict"] INFO: Requested target is [["@","","","both-noconflict"],{"BAR_BINS":["version","up","down"],"FOO_BINS":["version","ci","co"]}] INFO: Analysed target [["@","","","both-noconflict"],{"BAR_BINS":["version","up","down"],"FOO_BINS":["version","ci","co"]}] INFO: Discovered 2 actions, 1 tree overlays, 2 trees, 0 blobs INFO: Building [["@","","","both-noconflict"],{"BAR_BINS":["version","up","down"],"FOO_BINS":["version","ci","co"]}]. ERROR (action:aec6a6e881d3c3884420bee1330fbe9702ad3a7af11ee7f21fb59f88d6ba8f38): Tree-overlay computation failed: While merging the trees: - [394c2bd3bf4a07f8e22f6e73c2adcb03f28349df:30:t] - [e83b879ac94530f49855004138ea96df758b14d1:30:t] While merging the trees: - [7a5ed39406620e7b844788589f92fa4c17361c19:0:t] - [28891f0d46f5053861ac36bc7da18285e65ed267:0:t] Naming conflict detected at path "version": - [a263fff2a3b94429878303875861fd93bcdbe248:0:x] - [e9ebc93e1f91db5fab1fde7d22d8c6f59afab9f1:0:x] ERROR (action:aec6a6e881d3c3884420bee1330fbe9702ad3a7af11ee7f21fb59f88d6ba8f38): Failed to execute tree-overlay action. requested by - [["@","","","both-noconflict"],{"BAR_BINS":["version","up","down"],"FOO_BINS":["version","ci","co"]}]#0 inputs were - 394c2bd3bf4a07f8e22f6e73c2adcb03f28349df:30:t - e83b879ac94530f49855004138ea96df758b14d1:30:t ERROR: Build failed. ``` just-buildsystem-justbuild-b1fb5fa/etc/000077500000000000000000000000001516554100600203725ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/000077500000000000000000000000001516554100600222015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/000077500000000000000000000000001516554100600224665ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS000066400000000000000000000020531516554100600235220ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["TOOLCHAIN_CONFIG", "DEBUG"] , "base": [["@", "toolchain", "CC", "defaults"]] , "ADD_COMPILE_FLAGS": { "type": "case" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } , "case": { "msvc": { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["/Od", "/Z7"] , "else": ["/O2", "/DNDEBUG"] } } , "default": { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-O0", "-g"] , "else": ["-O2", "-DNDEBUG"] } } , "ADD_LDFLAGS": { "type": "if" , "cond": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "BUILD_STATIC" } , "then": ["-static"] } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.absl000066400000000000000000000074421516554100600244510ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CXXFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] ] , "body": { "type": "++" , "$1": [ ["-std=c++20"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "gnu": [ "-Wall" , "-Wextra" , "-Wcast-qual" , "-Wconversion-null" , "-Wformat-security" , "-Wmissing-declarations" , "-Woverlength-strings" , "-Wpointer-arith" , "-Wundef" , "-Wunused-local-typedefs" , "-Wunused-result" , "-Wvarargs" , "-Wvla" , "-Wwrite-strings" , "-DNOMINMAX" ] , "msvc": [ "/W3" , "/bigobj" , "/wd4005" , "/wd4068" , "/wd4180" , "/wd4244" , "/wd4267" , "/wd4503" , "/wd4800" , "/DNOMINMAX" , "/DWIN32_LEAN_AND_MEAN" , "/D_CRT_SECURE_NO_WARNINGS" , "/D_SCL_SECURE_NO_WARNINGS" , "/D_ENABLE_EXTENDED_ALIGNED_STORAGE" , "-ignore:4221" ] , "clang-cl": [ "/W3" , "/DNOMINMAX" , "/DWIN32_LEAN_AND_MEAN" , "/D_CRT_SECURE_NO_WARNINGS" , "/D_SCL_SECURE_NO_WARNINGS" , "/D_ENABLE_EXTENDED_ALIGNED_STORAGE" ] , "clang": [ "-Wall" , "-Wextra" , "-Wcast-qual" , "-Wconversion" , "-Wfloat-overflow-conversion" , "-Wfloat-zero-conversion" , "-Wfor-loop-analysis" , "-Wformat-security" , "-Wgnu-redeclared-enum" , "-Winfinite-recursion" , "-Winvalid-constexpr" , "-Wliteral-conversion" , "-Wmissing-declarations" , "-Woverlength-strings" , "-Wpointer-arith" , "-Wself-assign" , "-Wshadow-all" , "-Wshorten-64-to-32" , "-Wstring-conversion" , "-Wtautological-overlap-compare" , "-Wtautological-unsigned-zero-compare" , "-Wundef" , "-Wuninitialized" , "-Wunreachable-code" , "-Wunused-comparison" , "-Wunused-local-typedefs" , "-Wunused-result" , "-Wvla" , "-Wwrite-strings" , "-Wno-float-conversion" , "-Wno-implicit-float-conversion" , "-Wno-implicit-int-float-conversion" , "-Wno-sign-conversion" , "-Wno-unknown-warning-option" , "-DNOMINMAX" ] } , "default": [ "-Wall" , "-Wextra" , "-Wcast-qual" , "-Wconversion-null" , "-Wformat-security" , "-Wmissing-declarations" , "-Woverlength-strings" , "-Wpointer-arith" , "-Wundef" , "-Wunused-local-typedefs" , "-Wunused-result" , "-Wvarargs" , "-Wvla" , "-Wwrite-strings" , "-DNOMINMAX" ] } ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.archive000066400000000000000000000100221516554100600251350ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["OS", "TOOLCHAIN_CONFIG", "DEBUG", "ENABLE_BZip2", "HIDE_SYMBOLS"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , ["DEBUG", {"type": "var", "name": "DEBUG", "default": false}] ] , "body": { "type": "++" , "$1": [ ["-DHAVE_CONFIG_H"] , { "type": "if" , "cond": { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [["gnu", true], ["clang", true]] , "default": false } , "then": { "type": "++" , "$1": [ ["-Wall", "-Wformat", "-Wformat-security"] , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": [ "-g" , "-Wextra" , "-Wunused" , "-Wshadow" , "-Wmissing-prototypes" , "-Wcast-qual" ] } ] } } , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["darwin", ["-ffunction-sections -fdata-sections"]]] } , { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [ [ "xlc_r" , { "type": "++" , "$1": [ ["-qflag=e:e", "-qformat=sec"] , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-qhalt=w", "-g", "-qflag=w:w", "-qinfo=pro:use"] } ] } ] , [ "msvc" , { "type": "++" , "$1": [ ["/Oi"] , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": [ "/w14062" , "/w14254" , "/w14295" , "/w14296" , "/w14389" , "/w14505" , "/w14514" , "/w14702" , "/w14706" ] } , ["-D_CRT_SECURE_NO_DEPRECATE"] ] } ] , [ "mingw" , ["-D__USE_MINGW_ANSI_STDIO", "-D__MINGW_USE_VC2005_COMPAT"] ] ] } , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ ["hp-ux", ["-D_XOPEN_SOURCE=500"]] , ["mac", ["-Wno-deprecated-declarations"]] ] } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "msvc" } , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "HIDE_SYMBOLS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [ ["clang", ["-D__LIBARCHIVE_ENABLE_VISIBILITY"]] , ["gnu", ["-D__LIBARCHIVE_ENABLE_VISIBILITY"]] , ["intel", ["-D__LIBARCHIVE_ENABLE_VISIBILITY"]] ] } } } ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.boringssl000066400000000000000000000133551516554100600255320ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["TOOLCHAIN_CONFIG", "OS", "ARCH", "TARGET_ARCH"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "++" , "$1": [ ["-std=gnu17", "-DBORINGSSL_IMPLEMENTATION"] , { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "ARCH" , { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } ] , [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] , [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "posix_copts" , [ "-Wa,--noexecstack" , "-D_XOPEN_SOURCE=700" , "-Wall" , "-Wformat=2" , "-Wsign-compare" , "-Wmissing-field-initializers" , "-Wwrite-strings" , "-Wshadow" , "-fno-common" ] ] ] , "body": { "type": "++" , "$1": [ { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": {"arm": ["-Wno-atomic-alignment"]} } , { "type": "cond" , "cond": [ [ { "type": "or" , "$1": [ { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "linux_x86_64" } , { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "mac_x86_64" } ] } , {"type": "var", "name": "posix_copts"} ] , [ { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "windows_x86_64" } , ["-DWIN32_LEAN_AND_MEAN", "-DOPENSSL_NO_ASM"] ] ] , "default": ["-DOPENSSL_NO_ASM"] } , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "gnu": [ "-Wno-dangling-pointer" , "-Wno-array-bounds" , "-Wno-stringop-overflow" ] , "clang": [ "-Wno-unused-but-set-variable" , "-Wno-unknown-warning-option" ] } } ] } } ] } , "ADD_CXXFLAGS": { "type": "++" , "$1": [ ["-std=c++20", "-DBORINGSSL_IMPLEMENTATION"] , { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "ARCH" , { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } ] , [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] , [ "posix_copts" , [ "-Wa,--noexecstack" , "-D_XOPEN_SOURCE=700" , "-Wall" , "-Wformat=2" , "-Wsign-compare" , "-Wmissing-field-initializers" , "-Wwrite-strings" , "-Wshadow" , "-fno-common" , "-Wmissing-declarations" ] ] ] , "body": { "type": "cond" , "cond": [ [ { "type": "or" , "$1": [ { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "linux_x86_64" } , { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "mac_x86_64" } ] } , {"type": "var", "name": "posix_copts"} ] , [ { "type": "==" , "$1": {"type": "var", "name": "PLATFORM"} , "$2": "windows_x86_64" } , ["-DWIN32_LEAN_AND_MEAN", "-utf-8", "-DOPENSSL_NO_ASM"] ] ] , "default": ["-DOPENSSL_NO_ASM"] } } ] } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.bzip2000066400000000000000000000002271516554100600245500ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": ["-Wall", "-D_FILE_OFFSET_BITS=64"] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.curl000066400000000000000000000044031516554100600244670ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["TOOLCHAIN_CONFIG", "CURL_ENABLE_SSL", "ARCH", "TARGET_ARCH"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "CURL_ENABLE_SSL" , {"type": "var", "name": "CURL_ENABLE_SSL", "default": true} ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] ] , "body": { "type": "++" , "$1": [ { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [ [ "msvc" , [ "-D_CRT_SECURE_NO_DEPRECATE" , "-D_CRT_NONSTDC_NO_DEPRECATE" , "/W4" , "/MP" ] ] ] } , ["-DHAVE_CONFIG_H", "-DBUILDING_LIBCURL"] , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "msvc" } , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "CURL_HIDDEN_SYMBOLS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [ ["clang", ["-DCURL_HIDDEN_SYMBOLS"]] , ["gnu", ["-DCURL_HIDDEN_SYMBOLS"]] , ["sunpro", ["-DCURL_HIDDEN_SYMBOLS"]] , ["intel", ["-DCURL_HIDDEN_SYMBOLS"]] ] } } } , { "type": "case" , "expr": {"type": "var", "name": "TARGET_ARCH"} , "case": {"arm": ["-Wno-atomic-alignment"]} } ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.git2000066400000000000000000000055161516554100600243750ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["TOOLCHAIN_CONFIG", "OS"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "msvc": [ "-D_SCL_SECURE_NO_WARNINGS" , "-D_CRT_SECURE_NO_DEPRECATE" , "-D_CRT_NONSTDC_NO_DEPRECATE" , "/GF" , "/MP" , "/nologo" ] } , "default": { "type": "++" , "$1": [ [ "-std=gnu90" , "-D_GNU_SOURCE" , "-Wall" , "-Wextra" , "-DGIT_DEPRECATE_HARD" ] , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "sunos": [ "-D_POSIX_C_SOURCE=200112L" , "-D__EXTENSIONS__" , "-D_POSIX_PTHREAD_SEMANTICS" ] , "solaris": [ "-D_POSIX_C_SOURCE=200112L" , "-D__EXTENSIONS__" , "-D_POSIX_PTHREAD_SEMANTICS" ] } } , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "clang": [ "-Wdocumentation" , "-Wno-documentation-deprecated-sync" , "-Wno-unused-but-set-variable" , "-Wno-unused-but-set-parameter" , "-Wno-single-bit-bitfield-constant-conversion" ] , "gnu": ["-Wno-dangling-pointer"] } } , [ "-Wno-missing-field-initializers" , "-Wstrict-aliasing" , "-Wstrict-prototypes" , "-Wdeclaration-after-statement" , "-Wshift-count-overflow" , "-Wunused-const-variable" , "-Wunused-function" , "-Wint-conversion" , "-Wformat" , "-Wformat-security" , "-Wmissing-declarations" , "-Wno-implicit-fallthrough" , "-Wno-sign-compare" , "-Wno-unused-parameter" , "-Wno-uninitialized" , "-Wno-array-parameter" , "-Wno-unknown-warning-option" ] ] } } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.grpc000066400000000000000000000120021516554100600244470ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["DEBUG", "OS", "ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] ] , "body": { "type": "++" , "$1": [ ["-std=c11"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": {"msvc": []} , "default": [ "-Wall" , "-Wextra" , "-Wno-unused-parameter" , "-Wno-sign-compare" , "-Wno-implicit-fallthrough" , "-DOSATOMIC_USE_INLINED=1" , "-Wno-attributes" ] } , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "gnu": ["-Wno-clobbered", "-Wno-type-limits"] , "clang": ["-Wno-atomic-alignment"] } } , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-D_DEBUG", "-DDEBUG"] , "else": ["-Wframe-larger-than=16384"] } , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "windows": [ "-D_WIN32_WINNT=0x0600" , "-DWIN32_LEAN_AND_MEAN" , "-D_HAS_EXCEPTIONS=0" , "-DUNICODE" , "-D_UNICODE" , "-DNOMINMAX" ] } } ] } } , "ADD_CXXFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] ] , "body": { "type": "++" , "$1": [ ["-std=c++17"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": {"msvc": []} , "default": [ "-Wall" , "-Wextra" , "-Wno-unused-parameter" , "-Wno-uninitialized" , "-Wno-missing-field-initializers" , "-DOSATOMIC_USE_INLINED=1" , "-Wno-redundant-move" , "-Wno-comment" , "-Wno-attributes" , "-Wno-unused-function" , "-Wno-unknown-warning-option" , "-Wno-implicit-const-int-float-conversion" , "-Wno-stringop-overflow" , "-Wno-stringop-truncation" , "-Wno-class-memaccess" ] } , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-D_DEBUG", "-DDEBUG"] , "else": ["-Wframe-larger-than=16384"] } , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "gnu": [ "-Wno-array-bounds" , "-Wno-unused-variable" , "-Wno-dangling-pointer" , "-Wno-type-limits" ] } } , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "windows": [ "-D_WIN32_WINNT=0x0600" , "-DWIN32_LEAN_AND_MEAN" , "-D_HAS_EXCEPTIONS=0" , "-DUNICODE" , "-D_UNICODE" , "-DNOMINMAX" ] } } , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": {"arm": ["-Wno-atomic-alignment"]} } ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.just000066400000000000000000000004701516554100600245070ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": ["-std=gnu17"] , "ADD_CXXFLAGS": ["-std=c++20"] , "ADD_COMPILE_FLAGS": [ "-Wall" , "-Wextra" , "-Wpedantic" , "-Wsign-conversion" , "-Werror" , "-pedantic-errors" ] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.lzma000066400000000000000000000006441516554100600244700ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": [ "-std=c99" , "-Wall" , "-Wextra" , "-DHAVE_STDBOOL_H" , "-DHAVE__BOOL" , "-DHAVE_STDINT_H" , "-DHAVE_INTTYPES_H" , "-DHAVE_CHECK_CRC32" , "-D_GNU_SOURCE" , "-D__EXTENSIONS__" , "-D_POSIX_PTHREAD_SEMANTICS" , "-D_TANDEM_SOURCE" , "-D_ALL_SOURCE" ] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.protobuf000066400000000000000000000036671516554100600253750ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": ["-std=gnu17"] , "ADD_CXXFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] ] , "body": { "type": "++" , "$1": [ ["-std=c++17", "-DHAVE_PTHREAD=1", "-DHAVE_ZLIB=1"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "msvc": [ "/wd4065" , "/wd4146" , "/wd4244" , "/wd4251" , "/wd4267" , "/wd4305" , "/wd4307" , "/wd4309" , "/wd4334" , "/wd4355" , "/wd4506" , "/wd4800" , "/wd4996" ] } , "default": { "type": "++" , "$1": [ [ "-Wall" , "-Woverloaded-virtual" , "-Wno-sign-compare" , "-Wno-sign-conversion" , "-Wno-unused-function" , "-Wno-deprecated-declarations" , "-Wno-nonnull" ] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": { "gnu": ["-Wno-attributes"] , "clang": ["-Wno-atomic-alignment", "-Wno-return-stack-address"] } , "default": ["-Wno-deprecated-enum-enum-conversion"] } ] } } ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/TARGETS.re2000066400000000000000000000004031516554100600242060ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CXXFLAGS": [ "-std=c++11" , "-pthread" , "-Wall" , "-Wextra" , "-Wno-unused-parameter" , "-Wno-missing-field-initializers" ] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/auto/000077500000000000000000000000001516554100600234365ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/auto/TARGETS000066400000000000000000000000031516554100600244630ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/000077500000000000000000000000001516554100600244355ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS000066400000000000000000000000031516554100600254620ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.absl000066400000000000000000000000031516554100600264020ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.archive000066400000000000000000000000031516554100600271020ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.boringssl000066400000000000000000000000031516554100600274630ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.bzip2000066400000000000000000000000031516554100600265070ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.curl000066400000000000000000000000031516554100600264260ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.git2000066400000000000000000000000031516554100600263260ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.grpc000066400000000000000000000000031516554100600264140ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.just000066400000000000000000000000031516554100600264460ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.lzma000066400000000000000000000000031516554100600264240ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.protobuf000066400000000000000000000000031516554100600273210ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/pkgconfig/TARGETS.re2000066400000000000000000000000031516554100600261510ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/proto/000077500000000000000000000000001516554100600236315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/proto/TARGETS.grpc000066400000000000000000000002731516554100600256210ustar00rootroot00000000000000{ "defaults": { "type": ["CC/proto", "defaults"] , "PROTOC": ["bin/protoc"] , "deps": [["@", "protoc", "", "libprotobuf"]] , "toolchain": [["@", "protoc", "", "toolchain"]] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/proto/TARGETS.just000066400000000000000000000006321516554100600256520ustar00rootroot00000000000000{ "defaults": { "type": ["CC/proto", "defaults"] , "PROTOC": ["bin/protoc"] , "deps": [["@", "protoc", "", "libprotobuf"]] , "toolchain": [["@", "protoc", "", "toolchain"]] } , "service defaults": { "type": ["CC/proto", "defaults"] , "base": ["defaults"] , "GRPC_PLUGIN": ["bin/grpc_cpp_plugin"] , "deps": [["@", "grpc", "", "grpc++"]] , "toolchain": [["@", "grpc", "", "toolchain"]] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/test/000077500000000000000000000000001516554100600234455ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/test/TARGETS000066400000000000000000000000031516554100600244720ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/CC/test/TARGETS.just000066400000000000000000000000031516554100600254560ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/defaults/lint/000077500000000000000000000000001516554100600231475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/lint/TARGETS000066400000000000000000000001341516554100600242010ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["CC", "defaults"], ["shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/lint/TARGETS.just000066400000000000000000000001341516554100600251650ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["CC", "defaults"], ["shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/patch/000077500000000000000000000000001516554100600233005ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/patch/TARGETS000066400000000000000000000001541516554100600243340ustar00rootroot00000000000000{ "defaults": { "type": ["patch", "defaults"] , "base": [["@", "toolchain", "patch", "defaults"]] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/patch/TARGETS.absl000066400000000000000000000001401516554100600252470ustar00rootroot00000000000000{ "defaults": {"type": ["patch", "defaults"], "base": [["@", "base", "patch", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/patch/TARGETS.boringssl000066400000000000000000000001401516554100600263300ustar00rootroot00000000000000{ "defaults": {"type": ["patch", "defaults"], "base": [["@", "base", "patch", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/000077500000000000000000000000001516554100600233105ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS000066400000000000000000000001321516554100600243400ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "toolchain", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.absl000066400000000000000000000001251516554100600252620ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.archive000066400000000000000000000001251516554100600257620ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.bazel_remote_apis000066400000000000000000000001251516554100600300250ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.boringssl000066400000000000000000000001251516554100600263430ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.cares000066400000000000000000000001251516554100600254360ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.cli11000066400000000000000000000001251516554100600252520ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.curl000066400000000000000000000001251516554100600253060ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.fmt000066400000000000000000000001251516554100600251270ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.git2000066400000000000000000000001251516554100600252060ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.google_apis000066400000000000000000000001251516554100600266310ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.grpc000066400000000000000000000001251516554100600252740ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.gsl000066400000000000000000000001251516554100600251260ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.json000066400000000000000000000001251516554100600253120ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.just000066400000000000000000000001251516554100600253260ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.protobuf000066400000000000000000000001251516554100600262010ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.re2000066400000000000000000000001251516554100600250310ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/TARGETS.zlib000066400000000000000000000001251516554100600253010ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["@", "base", "shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/test/000077500000000000000000000000001516554100600242675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/test/TARGETS000066400000000000000000000002151516554100600253210ustar00rootroot00000000000000{ "summarizer": { "type": "summarizer" , "summarizer": [["FILE", null, "summarizer"]] , "artifacts": ["time-start", "time-stop"] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/test/TARGETS.bzip2000066400000000000000000000002151516554100600263460ustar00rootroot00000000000000{ "summarizer": { "type": "summarizer" , "summarizer": [["FILE", null, "summarizer"]] , "artifacts": ["time-start", "time-stop"] } } just-buildsystem-justbuild-b1fb5fa/etc/defaults/shell/test/TARGETS.just000066400000000000000000000002151516554100600263050ustar00rootroot00000000000000{ "summarizer": { "type": "summarizer" , "summarizer": [["FILE", null, "summarizer"]] , "artifacts": ["time-start", "time-stop"] } } just-buildsystem-justbuild-b1fb5fa/etc/dev/000077500000000000000000000000001516554100600211505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/dev/TARGETS000066400000000000000000000015671516554100600222150ustar00rootroot00000000000000{ "proto_bindings": { "type": ["@", "rules", "CC", "library"] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "googleapis", "", "google_bytestream_proto"] , ["@", "googleapis", "", "google_api_httpbody_proto"] , ["@", "googleapis", "", "google_api_expr_v1alpha1_checked_proto"] , ["@", "googleapis", "", "google_api_expr_v1alpha1_syntax_proto"] , ["src/buildtool/serve_api/serve_service", "just_serve_proto"] ] } , "just-ext-hdrs": { "type": ["@", "rules", "CC", "install-with-deps"] , "hdrs-only": ["yes"] , "targets": [ ["@", "ssl", "", "crypto"] , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["@", "fmt", "", "fmt"] , ["@", "cli11", "", "cli11"] , ["@", "json", "", "json"] , ["", "libgit2"] , ["", "libcurl"] , ["", "libarchive"] , "proto_bindings" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/000077500000000000000000000000001516554100600236525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.archive000066400000000000000000000001531516554100600263250ustar00rootroot00000000000000{ "archive": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libarchive"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.bazel_remote_apis000066400000000000000000000012061516554100600303700ustar00rootroot00000000000000{ "semver_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["semver_proto"] , "srcs": ["build/bazel/semver/semver.proto"] } , "remote_execution_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["remote_execution_proto"] , "service": ["yes"] , "srcs": ["build/bazel/remote/execution/v2/remote_execution.proto"] , "deps": [ "semver_proto" , ["@", "google_apis", "", "google_api_annotations_proto"] , ["@", "google_apis", "", "google_api_http_proto"] , ["@", "google_apis", "", "google_longrunning_operations_proto"] , ["@", "google_apis", "", "google_rpc_status_proto"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.boringssl000066400000000000000000000003121516554100600267030ustar00rootroot00000000000000{ "ssl": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libssl"] } , "crypto": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libcrypto"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.cares000066400000000000000000000001461516554100600260030ustar00rootroot00000000000000{ "ares": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libcares"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.cli11000066400000000000000000000001351516554100600256150ustar00rootroot00000000000000{ "cli11": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["CLI11"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.curl000066400000000000000000000001451516554100600256520ustar00rootroot00000000000000{ "curl": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libcurl"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.fmt000066400000000000000000000001311516554100600254660ustar00rootroot00000000000000{ "fmt": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["fmt"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.git2000066400000000000000000000001451516554100600255520ustar00rootroot00000000000000{ "git2": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libgit2"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.google_apis000066400000000000000000000041051516554100600271750ustar00rootroot00000000000000{ "google_api_http_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_http_proto"] , "srcs": ["google/api/http.proto"] } , "google_api_httpbody_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_httpbody_proto"] , "srcs": ["google/api/httpbody.proto"] } , "google_api_annotations_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_annotations_proto"] , "srcs": ["google/api/annotations.proto"] , "deps": ["google_api_http_proto"] } , "google_api_client_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_client_proto"] , "srcs": ["google/api/client.proto"] , "deps": ["google_api_launch_stage_proto"] } , "google_api_launch_stage_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_launch_stage_proto"] , "srcs": ["google/api/launch_stage.proto"] } , "google_api_expr_v1alpha1_checked_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_expr_v1alpha1_checked_proto"] , "srcs": ["google/api/expr/v1alpha1/checked.proto"] , "deps": ["google_api_expr_v1alpha1_syntax_proto"] } , "google_api_expr_v1alpha1_syntax_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_expr_v1alpha1_syntax_proto"] , "srcs": ["google/api/expr/v1alpha1/syntax.proto"] } , "google_bytestream_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_bytestream_proto"] , "service": ["yes"] , "srcs": ["google/bytestream/bytestream.proto"] , "deps": ["google_api_annotations_proto"] } , "google_rpc_status_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_rpc_status_proto"] , "srcs": ["google/rpc/status.proto"] } , "google_longrunning_operations_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["google_longrunning_operations_proto"] , "service": ["yes"] , "srcs": ["google/longrunning/operations.proto"] , "deps": [ "google_api_annotations_proto" , "google_api_client_proto" , "google_rpc_status_proto" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.grpc000066400000000000000000000006511516554100600256420ustar00rootroot00000000000000{ "grpc_cpp_plugin": {"type": "install", "deps": [["bin", "grpc_cpp_plugin"]]} , "grpc++": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["grpc++"] } , "grpc": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["grpc"]} , "gpr": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["gpr"]} , "toolchain": {"type": "install", "deps": ["bin/grpc_cpp_plugin"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.gsl000066400000000000000000000001311516554100600254650ustar00rootroot00000000000000{ "gsl": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["gsl"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.json000066400000000000000000000001531516554100600256550ustar00rootroot00000000000000{ "json": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["nlohmann_json"] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.protobuf000066400000000000000000000010201516554100600265360ustar00rootroot00000000000000{ "protoc": {"type": "install", "deps": [["bin", "protoc"]]} , "libprotoc": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["libprotoc"] } , "libprotobuf": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["protobuf"] } , "libprotobuf_lite": { "type": ["@", "rules", "CC/pkgconfig", "system_library"] , "name": ["protobuf-lite"] } , "toolchain": { "type": "install" , "files": {"bin/protoc": "protoc"} , "deps": [["TREE", null, "include/google/protobuf"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.re2000066400000000000000000000001311516554100600253700ustar00rootroot00000000000000{ "re2": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["re2"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/TARGETS.zlib000066400000000000000000000001331516554100600256420ustar00rootroot00000000000000{ "zlib": {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["zlib"]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/bin/000077500000000000000000000000001516554100600244225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/bin/TARGETS.grpc000066400000000000000000000001321516554100600264040ustar00rootroot00000000000000{ "grpc_cpp_plugin": {"type": "install", "deps": [["FILE", null, "grpc_cpp_plugin"]]} } just-buildsystem-justbuild-b1fb5fa/etc/import.pkgconfig/bin/TARGETS.protobuf000066400000000000000000000001041516554100600273100ustar00rootroot00000000000000{"protoc": {"type": "install", "deps": [["FILE", null, "protoc"]]}} just-buildsystem-justbuild-b1fb5fa/etc/import/000077500000000000000000000000001516554100600217045ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/RULES.grpc000066400000000000000000000154071516554100600234620ustar00rootroot00000000000000{ "combined lib": { "doc": ["Combine multiple static libraries into a single one."] , "target_fields": ["deps"] , "string_fields": ["name", "stage"] , "config_vars": ["AR", "ENV"] , "implicit": {"defaults": [["@", "rules", "CC", "defaults"]]} , "field_doc": { "name": ["The name of the library (without leading \"lib\" or trailing \".a\")."] , "deps": ["All libraries (includes transitive) that should be combined."] , "stage": [ "The logical location of all public headers and the resulting library" , "file. Individual directory components are joined with \"/\"." ] } , "config_doc": { "AR": [ "The archive tool to used for creating the library. If None, the" , "respective value from [\"@\", \"rules\", \"CC\", \"defaults\"] will" , "be taken." ] , "ENV": ["The environment for any action generated."] } , "imports": { "compile-deps": ["@", "rules", "CC", "compile-deps"] , "compile-args-deps": ["@", "rules", "CC", "compile-args-deps"] , "link-deps": ["@", "rules", "CC", "link-deps"] , "link-args-deps": ["@", "rules", "CC", "link-args-deps"] , "cflags-files-deps": ["@", "rules", "CC", "cflags-files-deps"] , "ldflags-files-deps": ["@", "rules", "CC", "ldflags-files-deps"] , "default-AR": ["@", "rules", "CC", "default-AR"] } , "expression": { "type": "let*" , "bindings": [ [ "AR" , { "type": "var" , "name": "AR" , "default": {"type": "CALL_EXPRESSION", "name": "default-AR"} } ] , ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "libname" , { "type": "join" , "$1": ["lib", {"type": "var", "name": "name"}, ".a"] } ] , [ "libpath" , { "type": "if" , "cond": {"type": "var", "name": "stage"} , "then": { "type": "join" , "separator": "/" , "$1": [ {"type": "var", "name": "stage"} , {"type": "var", "name": "libname"} ] } , "else": {"type": "var", "name": "libname"} } ] , ["deps-fieldnames", ["deps"]] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] , ["unstaged libs", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , [ "libs" , { "type": "to_subdir" , "subdir": "libs" , "$1": {"type": "var", "name": "unstaged libs"} } ] , ["link-deps", {"type": "empty_map"}] , [ "link-args-deps" , {"type": "CALL_EXPRESSION", "name": "link-args-deps"} ] , [ "non-lib-link-args-deps" , { "type": "++" , "$1": { "type": "foreach" , "var": "arg" , "range": {"type": "var", "name": "link-args-deps"} , "body": { "type": "if" , "cond": { "type": "lookup" , "map": {"type": "var", "name": "unstaged libs"} , "key": {"type": "var", "name": "arg"} } , "then": [] , "else": [{"type": "var", "name": "arg"}] } } } ] , [ "link-args" , { "type": "++" , "$1": [ [{"type": "var", "name": "libpath"}] , {"type": "var", "name": "non-lib-link-args-deps"} ] } ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , [ "package" , { "type": "let*" , "bindings": [["name", {"type": "var", "name": "name"}]] , "body": {"type": "env", "vars": ["name", "cflags-files", "ldflags-files"]} } ] , [ "combine.sh" , { "type": "singleton_map" , "key": "combine.sh" , "value": { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -eu" , "readonly ROOT=$(pwd)" , "readonly AR=$1" , "readonly OUT=$2" , "shift 2" , "for l in $@; do" , " OUTDIR=${ROOT}/objs/$l" , " mkdir -p $OUTDIR" , " cd $OUTDIR" , " $AR -t $ROOT/$l | sort | uniq -c | sed -e 's/^\\s*//' > obj.list" , " NUM=$(wc -l obj.list | cut -d' ' -f1)" , " for o in $(seq 1 $NUM); do" , " CNT=$(sed ''$o'q;d' obj.list | cut -d' ' -f1)" , " OBJ=$(sed ''$o'q;d' obj.list | cut -d' ' -f2)" , " for i in $(seq 1 $CNT); do" , " mkdir -p $i" , " $AR -xN $i $ROOT/$l $OBJ" , " mv *.o $i/" , " done" , " done" , " cd - >/dev/null" , " $AR -qc $OUT $(find $OUTDIR -type f -name *.o | sort)" , "done" , "$AR s $OUT" ] } } } ] , [ "combined lib" , { "type": "ACTION" , "outs": [{"type": "var", "name": "libname"}] , "inputs": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "libs"} , {"type": "var", "name": "combine.sh"} ] } , "cmd": { "type": "++" , "$1": [ [ "sh" , "combine.sh" , {"type": "var", "name": "AR"} , {"type": "var", "name": "libname"} ] , {"type": "keys", "$1": {"type": "var", "name": "libs"}} ] } , "env": {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "combined lib"} , "provides": { "type": "map_union" , "$1": [ { "type": "env" , "vars": [ "compile-deps" , "compile-args" , "link-deps" , "link-args" , "package" ] } , { "type": "var" , "name": "extra-provides" , "default": {"type": "empty_map"} } ] } } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.archive000066400000000000000000000266771516554100600244020ustar00rootroot00000000000000{ "archive": { "type": "export" , "target": "archive_config" , "doc": ["The Archive linkable library"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "ENABLE_MBEDTLS" , "ENABLE_NETTLE" , "ENABLE_OPENSSL" , "ENABLE_LIBB2" , "ENABLE_LZ4" , "ENABLE_LZO" , "ENABLE_LZMA" , "ENABLE_ZSTD" , "ENABLE_ZLIB" , "ENABLE_BZip2" , "ENABLE_LIBXML2" , "ENABLE_EXPAT" , "ENABLE_PCREPOSIX" , "ENABLE_PCRE2POSIX" , "ENABLE_LIBGCC" , "ENABLE_CNG" , "ENABLE_XATTR" , "ENABLE_ACL" , "ENABLE_ICONV" , "ENABLE_LIBMD" , "ENABLE_PCRE" , "ENABLE_PCRE2" , "ENABLE_REGEX" , "XATTR_PROVIDER" , "ENABLE_RICHACL" , "USE_NFS4" , "HIDE_SYMBOLS" , "PKG_CONFIG_ARGS" ] , "config_doc": { "ENABLE_MBEDTLS": ["Boolean. Link against mbedtls (system or open name)."] , "ENABLE_NETTLE": ["Boolean. Link against nettle (system or open name)."] , "ENABLE_OPENSSL": [ "Boolean. Default value: true." , "Link against OpenSSL (system or open name)." ] , "ENABLE_LIBB2": [ "Boolean. Default value: true. Link against libb2 (system or open name)." ] , "ENABLE_LZ4": ["Boolean. Default value: true. Link against lz4 (system or open name)."] , "ENABLE_LZO": ["Boolean. Link against lzo (system or open name)."] , "ENABLE_LZMA": [ "Boolean. Default value: true. Link against lzma (system or open name)." ] , "ENABLE_ZSTD": [ "Boolean. Default value: true. Link against zstd (system or open name)." ] , "ENABLE_ZLIB": [ "Boolean. Default value: true. Link against zlib (system or open name)." ] , "ENABLE_BZip2": [ "Boolean. Default value: true. Link against libbz2 (system or open name)." ] , "ENABLE_LIBXML2": [ "Boolean. Default value: true. Link against libxml2 (system or open name)." , "Value used only if ENABLE_ICONV==true." ] , "ENABLE_EXPAT": [ "Boolean. Default value: true. Link against EXPAT (system or open name)." ] , "ENABLE_PCREPOSIX": [ "Boolean. Default value: true. Link against PCREPOSIX (system or open name)." ] , "ENABLE_PCRE2POSIX": [ "Boolean. Default value: true. Link against PCRE2POSIX (system or open name)." ] , "ENABLE_LIBGCC": [ "Boolean. Default value: true. Link against LIBGCC (system or open name)." ] , "ENABLE_CNG": [ "Boolean. Use CNG (Crypto Next Generation)." , "Default value: true, if ~OS==\"windows\"~." , "Disabled if not ~OS==\"windows\"~." ] , "ENABLE_XATTR": ["Boolean. Default value: true. Enable extended attribute support."] , "ENABLE_ACL": ["Boolean. Default value: true. Enable ACL support."] , "ENABLE_ICONV": ["Boolean. Default value: true. Enable iconv support."] , "ENABLE_LIBMD": [ "Boolean. Link against libmd (system or open name)." , "Disabled if ~ENABLE_OPENSSL==true~." ] , "ENABLE_PCRE": ["Boolean. Link against pcre (system or open name)."] , "ENABLE_PCRE2": ["Boolean. Link against pcre2 (system or open name)."] , "ENABLE_REGEX": ["Boolean. Link against regex (system or open name)."] , "XATTR_PROVIDER": [ "\"attr\": Link against attr (system or open name) for xattr support." , "\"gnu\": Xattr support provided natively." , "\"none\"|null: No xattr support. Build fails if ~ENABLE_XATTR==true~." ] , "ENABLE_RICHACL": ["Boolean. Link against RichACL (system or open name)."] , "USE_NSF4": ["Boolean. Explicitly give the NSF4 support state for the target OS."] , "HIDE_SYMBOLS": [ "Boolean. Default value: true." , "Hide all symbols not officially external." , "Has effect if" , "TOOLCHAIN_CONFIG[\"FAMILY\"] == \"clang\"|\"gnu\"|\"intel\"." ] } } , "archive_config": { "type": "configure" , "arguments_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "HIDE_SYMBOLS" , "ENABLE_MBEDTLS" , "ENABLE_NETTLE" , "ENABLE_OPENSSL" , "ENABLE_LIBB2" , "ENABLE_LZ4" , "ENABLE_LZO" , "ENABLE_LZMA" , "ENABLE_ZSTD" , "ENABLE_ZLIB" , "ENABLE_BZip2" , "ENABLE_LIBXML2" , "ENABLE_EXPAT" , "ENABLE_PCREPOSIX" , "ENABLE_PCRE2POSIX" , "ENABLE_LIBGCC" , "ENABLE_CNG" , "ENABLE_XATTR" , "ENABLE_ACL" , "ENABLE_ICONV" , "ENABLE_LIBMD" , "ENABLE_PCRE" , "ENABLE_PCRE2" , "ENABLE_REGEX" , "XATTR_PROVIDER" , "ENABLE_RICHACL" , "USE_NFS4" ] , "target": ["./", "libarchive", "libarchive"] , "config": { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] , [ "ENABLE_MBEDTLS" , {"type": "var", "name": "ENABLE_MBEDTLS", "default": false} ] , [ "ENABLE_NETTLE" , {"type": "var", "name": "ENABLE_NETTLE", "default": false} ] , [ "ENABLE_OPENSSL" , { "type": "case*" , "expr": {"type": "var", "name": "ENABLE_OPENSSL"} , "case": [ [ true , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "darwin" } , "then": false , "else": true } ] ] , "default": { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "darwin" } , "then": false , "else": true } } ] , [ "ENABLE_LIBB2" , {"type": "var", "name": "ENABLE_LIBB2", "default": true} ] , ["ENABLE_LZ4", {"type": "var", "name": "ENABLE_LZ4", "default": true}] , ["ENABLE_LZO", {"type": "var", "name": "ENABLE_LZO", "default": false}] , [ "ENABLE_LZMA" , {"type": "var", "name": "ENABLE_LZMA", "default": true} ] , [ "ENABLE_ZSTD" , {"type": "var", "name": "ENABLE_ZSTD", "default": true} ] , [ "ENABLE_ZLIB" , {"type": "var", "name": "ENABLE_ZLIB", "default": true} ] , [ "ENABLE_BZip2" , {"type": "var", "name": "ENABLE_BZip2", "default": true} ] , [ "ENABLE_LIBXML2" , { "type": "case*" , "expr": {"type": "var", "name": "ENABLE_LIBXML2"} , "case": [[true, {"type": "var", "name": "ENABLE_ICONV"}]] , "default": {"type": "var", "name": "ENABLE_ICONV"} } ] , [ "ENABLE_EXPAT" , { "type": "case*" , "expr": {"type": "var", "name": "ENABLE_EXPAT"} , "case": [ [ true , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBXML2"} , "then": false , "else": true } ] ] , "default": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBXML2"} , "then": false , "else": true } } ] , [ "ENABLE_PCREPOSIX" , {"type": "var", "name": "ENABLE_PCREPOSIX", "default": true} ] , [ "ENABLE_PCRE2POSIX" , {"type": "var", "name": "ENABLE_PCREPOSIX", "default": true} ] , [ "ENABLE_LIBGCC" , {"type": "var", "name": "ENABLE_LIBGCC", "default": true} ] , [ "ENABLE_CNG" , { "type": "case*" , "expr": {"type": "var", "name": "ENABLE_CNG"} , "case": [ [ true , { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } ] ] , "default": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } } ] , [ "ENABLE_XATTR" , {"type": "var", "name": "ENABLE_XATTR", "default": true} ] , ["ENABLE_ACL", {"type": "var", "name": "ENABLE_ACL", "default": true}] , [ "ENABLE_ICONV" , {"type": "var", "name": "ENABLE_ICONV", "default": true} ] , [ "ENABLE_LIBMD" , { "type": "case*" , "expr": {"type": "var", "name": "ENABLE_LIBMD"} , "case": [ [ true , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": false , "else": true } ] ] , "default": false } ] , [ "ENABLE_PCRE" , {"type": "var", "name": "ENABLE_PCRE", "default": false} ] , [ "ENABLE_PCRE2" , {"type": "var", "name": "ENABLE_PCRE2", "default": false} ] , [ "ENABLE_REGEX" , {"type": "var", "name": "ENABLE_REGEX", "default": false} ] , [ "XATTR_PROVIDER" , { "type": "case*" , "expr": {"type": "var", "name": "XATTR_PROVIDER"} , "case": [ [ "none" , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": { "type": "fail" , "msg": "Extended attributes support was enabled, but providing library not specified." } , "else": "none" } ] ] , "default": {"type": "var", "name": "XATTR_PROVIDER"} } ] , [ "ENABLE_RICHACL" , {"type": "var", "name": "ENABLE_RICHACL", "default": false} ] , ["USE_NFS4", {"type": "var", "name": "USE_NFS4", "default": true}] , [ "HIDE_SYMBOLS" , {"type": "var", "name": "HIDE_SYMBOLS", "default": true} ] ] , "body": { "type": "env" , "vars": [ "USE_SYSTEM_LIBS" , "DEBUG" , "ENABLE_MBEDTLS" , "ENABLE_NETTLE" , "ENABLE_OPENSSL" , "ENABLE_LIBB2" , "ENABLE_LZ4" , "ENABLE_LZO" , "ENABLE_LZMA" , "ENABLE_ZSTD" , "ENABLE_ZLIB" , "ENABLE_BZip2" , "ENABLE_LIBXML2" , "ENABLE_EXPAT" , "ENABLE_PCREPOSIX" , "ENABLE_PCRE2POSIX" , "ENABLE_LIBGCC" , "ENABLE_CNG" , "ENABLE_XATTR" , "ENABLE_ACL" , "ENABLE_ICONV" , "ENABLE_LIBMD" , "ENABLE_PCRE" , "ENABLE_PCRE2" , "ENABLE_REGEX" , "XATTR_PROVIDER" , "ENABLE_RICHACL" , "USE_NFS4" , "HIDE_SYMBOLS" ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.bazel_remote_apis000066400000000000000000000021431516554100600264230ustar00rootroot00000000000000{ "semver_proto": {"type": "export", "target": "semver_proto (unexported)"} , "semver_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["semver_proto"] , "srcs": ["build/bazel/semver/semver.proto"] } , "remote_execution_proto impl": { "type": ["@", "rules", "proto", "library"] , "name": ["remote_execution_proto"] , "service": ["yes"] , "srcs": ["build/bazel/remote/execution/v2/remote_execution.proto"] , "deps": [ "semver_proto" , ["@", "google_apis", "", "google_api_annotations_proto"] , ["@", "google_apis", "", "google_api_http_proto"] , ["@", "google_apis", "", "google_longrunning_operations_proto"] , ["@", "google_apis", "", "google_rpc_status_proto"] ] } , "remote_execution_proto": { "type": "export" , "target": "remote_execution_proto impl" , "doc": [ "Remote Execution API" , "" , "The Remote Execution API is an API that, at its most general, allows clients" , "to request execution of binaries on a remote system." ] , "flexible_config": ["ARCH", "ENV", "HOST_ARCH", "PATCH", "TOOLCHAIN_CONFIG"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.boringssl000066400000000000000000000566261516554100600247600ustar00rootroot00000000000000{ "crypto": { "type": "configure" , "target": "exported crypto" , "arguments_config": ["ARCH", "TARGET_ARCH"] , "config": { "type": "let*" , "bindings": [ [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["TARGET_ARCH"]} } } , "exported crypto": { "type": "export" , "target": "crypto-lib" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" ] } , "ssl": { "type": "export" , "target": "ssl-lib" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" ] } , "crypto-lib": { "type": ["@", "rules", "CC", "library"] , "name": ["crypto"] , "pkg-name": ["libcrypto"] , "arguments_config": ["OS", "ARCH", "TARGET_ARCH"] , "hdrs": [["./", "src/include/openssl", "crypto_headers"]] , "private-hdrs": ["bcm_internal_headers", "crypto_internal_headers"] , "srcs": ["bcm_sources", "crypto_sources", "bcm_sources_asm", "crypto_sources_asm"] , "private-ldflags": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "OS"}, "$2": "windows"} , "then": ["-defaultlib:advapi32.lib", "-defaultlib:ws2_32.lib"] } , ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] ] } } , "ssl-lib": { "type": ["@", "rules", "CC", "library"] , "name": ["ssl"] , "pkg-name": ["libssl"] , "hdrs": [["./", "src/include/openssl", "ssl_headers"]] , "private-hdrs": ["ssl_internal_headers", "crypto_internal_headers"] , "srcs": ["ssl_sources"] , "deps": ["crypto"] } , "bcm_sources": {"type": "install", "deps": ["src/crypto/fipsmodule/bcm.cc"]} , "bcm_internal_headers": { "type": "install" , "deps": [ "src/crypto/fipsmodule/aes/aes.cc.inc" , "src/crypto/fipsmodule/aes/aes_nohw.cc.inc" , "src/crypto/fipsmodule/aes/key_wrap.cc.inc" , "src/crypto/fipsmodule/aes/mode_wrappers.cc.inc" , "src/crypto/fipsmodule/bn/add.cc.inc" , "src/crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc" , "src/crypto/fipsmodule/bn/bn.cc.inc" , "src/crypto/fipsmodule/bn/bytes.cc.inc" , "src/crypto/fipsmodule/bn/cmp.cc.inc" , "src/crypto/fipsmodule/bn/ctx.cc.inc" , "src/crypto/fipsmodule/bn/div.cc.inc" , "src/crypto/fipsmodule/bn/div_extra.cc.inc" , "src/crypto/fipsmodule/bn/exponentiation.cc.inc" , "src/crypto/fipsmodule/bn/gcd.cc.inc" , "src/crypto/fipsmodule/bn/gcd_extra.cc.inc" , "src/crypto/fipsmodule/bn/generic.cc.inc" , "src/crypto/fipsmodule/bn/jacobi.cc.inc" , "src/crypto/fipsmodule/bn/montgomery.cc.inc" , "src/crypto/fipsmodule/bn/montgomery_inv.cc.inc" , "src/crypto/fipsmodule/bn/mul.cc.inc" , "src/crypto/fipsmodule/bn/prime.cc.inc" , "src/crypto/fipsmodule/bn/random.cc.inc" , "src/crypto/fipsmodule/bn/rsaz_exp.cc.inc" , "src/crypto/fipsmodule/bn/shift.cc.inc" , "src/crypto/fipsmodule/bn/sqrt.cc.inc" , "src/crypto/fipsmodule/cipher/aead.cc.inc" , "src/crypto/fipsmodule/cipher/cipher.cc.inc" , "src/crypto/fipsmodule/cipher/e_aes.cc.inc" , "src/crypto/fipsmodule/cipher/e_aesccm.cc.inc" , "src/crypto/fipsmodule/cmac/cmac.cc.inc" , "src/crypto/fipsmodule/dh/check.cc.inc" , "src/crypto/fipsmodule/dh/dh.cc.inc" , "src/crypto/fipsmodule/digest/digest.cc.inc" , "src/crypto/fipsmodule/digest/digests.cc.inc" , "src/crypto/fipsmodule/digestsign/digestsign.cc.inc" , "src/crypto/fipsmodule/ec/ec.cc.inc" , "src/crypto/fipsmodule/ec/ec_key.cc.inc" , "src/crypto/fipsmodule/ec/ec_montgomery.cc.inc" , "src/crypto/fipsmodule/ec/felem.cc.inc" , "src/crypto/fipsmodule/ec/oct.cc.inc" , "src/crypto/fipsmodule/ec/p224-64.cc.inc" , "src/crypto/fipsmodule/ec/p256-nistz.cc.inc" , "src/crypto/fipsmodule/ec/p256.cc.inc" , "src/crypto/fipsmodule/ec/scalar.cc.inc" , "src/crypto/fipsmodule/ec/simple.cc.inc" , "src/crypto/fipsmodule/ec/simple_mul.cc.inc" , "src/crypto/fipsmodule/ec/util.cc.inc" , "src/crypto/fipsmodule/ec/wnaf.cc.inc" , "src/crypto/fipsmodule/ecdh/ecdh.cc.inc" , "src/crypto/fipsmodule/ecdsa/ecdsa.cc.inc" , "src/crypto/fipsmodule/hkdf/hkdf.cc.inc" , "src/crypto/fipsmodule/hmac/hmac.cc.inc" , "src/crypto/fipsmodule/modes/cbc.cc.inc" , "src/crypto/fipsmodule/modes/cfb.cc.inc" , "src/crypto/fipsmodule/modes/ctr.cc.inc" , "src/crypto/fipsmodule/modes/gcm.cc.inc" , "src/crypto/fipsmodule/modes/gcm_nohw.cc.inc" , "src/crypto/fipsmodule/modes/ofb.cc.inc" , "src/crypto/fipsmodule/modes/polyval.cc.inc" , "src/crypto/fipsmodule/rand/ctrdrbg.cc.inc" , "src/crypto/fipsmodule/rand/rand.cc.inc" , "src/crypto/fipsmodule/rsa/blinding.cc.inc" , "src/crypto/fipsmodule/rsa/padding.cc.inc" , "src/crypto/fipsmodule/rsa/rsa.cc.inc" , "src/crypto/fipsmodule/rsa/rsa_impl.cc.inc" , "src/crypto/fipsmodule/self_check/fips.cc.inc" , "src/crypto/fipsmodule/self_check/self_check.cc.inc" , "src/crypto/fipsmodule/service_indicator/service_indicator.cc.inc" , "src/crypto/fipsmodule/sha/sha1.cc.inc" , "src/crypto/fipsmodule/sha/sha256.cc.inc" , "src/crypto/fipsmodule/sha/sha512.cc.inc" , "src/crypto/fipsmodule/tls/kdf.cc.inc" ] } , "bcm_sources_asm": { "type": "install" , "deps": [ "src/gen/bcm/aes-gcm-avx10-x86_64-apple.S" , "src/gen/bcm/aes-gcm-avx10-x86_64-linux.S" , "src/gen/bcm/aesni-gcm-x86_64-apple.S" , "src/gen/bcm/aesni-gcm-x86_64-linux.S" , "src/gen/bcm/aesni-x86-apple.S" , "src/gen/bcm/aesni-x86-linux.S" , "src/gen/bcm/aesni-x86_64-apple.S" , "src/gen/bcm/aesni-x86_64-linux.S" , "src/gen/bcm/aesv8-armv7-linux.S" , "src/gen/bcm/aesv8-armv8-apple.S" , "src/gen/bcm/aesv8-armv8-linux.S" , "src/gen/bcm/aesv8-armv8-win.S" , "src/gen/bcm/aesv8-gcm-armv8-apple.S" , "src/gen/bcm/aesv8-gcm-armv8-linux.S" , "src/gen/bcm/aesv8-gcm-armv8-win.S" , "src/gen/bcm/armv4-mont-linux.S" , "src/gen/bcm/armv8-mont-apple.S" , "src/gen/bcm/armv8-mont-linux.S" , "src/gen/bcm/armv8-mont-win.S" , "src/gen/bcm/bn-586-apple.S" , "src/gen/bcm/bn-586-linux.S" , "src/gen/bcm/bn-armv8-apple.S" , "src/gen/bcm/bn-armv8-linux.S" , "src/gen/bcm/bn-armv8-win.S" , "src/gen/bcm/bsaes-armv7-linux.S" , "src/gen/bcm/co-586-apple.S" , "src/gen/bcm/co-586-linux.S" , "src/gen/bcm/ghash-armv4-linux.S" , "src/gen/bcm/ghash-neon-armv8-apple.S" , "src/gen/bcm/ghash-neon-armv8-linux.S" , "src/gen/bcm/ghash-neon-armv8-win.S" , "src/gen/bcm/ghash-ssse3-x86-apple.S" , "src/gen/bcm/ghash-ssse3-x86-linux.S" , "src/gen/bcm/ghash-ssse3-x86_64-apple.S" , "src/gen/bcm/ghash-ssse3-x86_64-linux.S" , "src/gen/bcm/ghash-x86-apple.S" , "src/gen/bcm/ghash-x86-linux.S" , "src/gen/bcm/ghash-x86_64-apple.S" , "src/gen/bcm/ghash-x86_64-linux.S" , "src/gen/bcm/ghashv8-armv7-linux.S" , "src/gen/bcm/ghashv8-armv8-apple.S" , "src/gen/bcm/ghashv8-armv8-linux.S" , "src/gen/bcm/ghashv8-armv8-win.S" , "src/gen/bcm/p256-armv8-asm-apple.S" , "src/gen/bcm/p256-armv8-asm-linux.S" , "src/gen/bcm/p256-armv8-asm-win.S" , "src/gen/bcm/p256-x86_64-asm-apple.S" , "src/gen/bcm/p256-x86_64-asm-linux.S" , "src/gen/bcm/p256_beeu-armv8-asm-apple.S" , "src/gen/bcm/p256_beeu-armv8-asm-linux.S" , "src/gen/bcm/p256_beeu-armv8-asm-win.S" , "src/gen/bcm/p256_beeu-x86_64-asm-apple.S" , "src/gen/bcm/p256_beeu-x86_64-asm-linux.S" , "src/gen/bcm/rdrand-x86_64-apple.S" , "src/gen/bcm/rdrand-x86_64-linux.S" , "src/gen/bcm/rsaz-avx2-apple.S" , "src/gen/bcm/rsaz-avx2-linux.S" , "src/gen/bcm/sha1-586-apple.S" , "src/gen/bcm/sha1-586-linux.S" , "src/gen/bcm/sha1-armv4-large-linux.S" , "src/gen/bcm/sha1-armv8-apple.S" , "src/gen/bcm/sha1-armv8-linux.S" , "src/gen/bcm/sha1-armv8-win.S" , "src/gen/bcm/sha1-x86_64-apple.S" , "src/gen/bcm/sha1-x86_64-linux.S" , "src/gen/bcm/sha256-586-apple.S" , "src/gen/bcm/sha256-586-linux.S" , "src/gen/bcm/sha256-armv4-linux.S" , "src/gen/bcm/sha256-armv8-apple.S" , "src/gen/bcm/sha256-armv8-linux.S" , "src/gen/bcm/sha256-armv8-win.S" , "src/gen/bcm/sha256-x86_64-apple.S" , "src/gen/bcm/sha256-x86_64-linux.S" , "src/gen/bcm/sha512-586-apple.S" , "src/gen/bcm/sha512-586-linux.S" , "src/gen/bcm/sha512-armv4-linux.S" , "src/gen/bcm/sha512-armv8-apple.S" , "src/gen/bcm/sha512-armv8-linux.S" , "src/gen/bcm/sha512-armv8-win.S" , "src/gen/bcm/sha512-x86_64-apple.S" , "src/gen/bcm/sha512-x86_64-linux.S" , "src/gen/bcm/vpaes-armv7-linux.S" , "src/gen/bcm/vpaes-armv8-apple.S" , "src/gen/bcm/vpaes-armv8-linux.S" , "src/gen/bcm/vpaes-armv8-win.S" , "src/gen/bcm/vpaes-x86-apple.S" , "src/gen/bcm/vpaes-x86-linux.S" , "src/gen/bcm/vpaes-x86_64-apple.S" , "src/gen/bcm/vpaes-x86_64-linux.S" , "src/gen/bcm/x86-mont-apple.S" , "src/gen/bcm/x86-mont-linux.S" , "src/gen/bcm/x86_64-mont-apple.S" , "src/gen/bcm/x86_64-mont-linux.S" , "src/gen/bcm/x86_64-mont5-apple.S" , "src/gen/bcm/x86_64-mont5-linux.S" , "src/third_party/fiat/asm/fiat_p256_adx_mul.S" , "src/third_party/fiat/asm/fiat_p256_adx_sqr.S" ] } , "crypto_sources": { "type": "install" , "deps": [ "src/crypto/asn1/a_bitstr.cc" , "src/crypto/asn1/a_bool.cc" , "src/crypto/asn1/a_d2i_fp.cc" , "src/crypto/asn1/a_dup.cc" , "src/crypto/asn1/a_gentm.cc" , "src/crypto/asn1/a_i2d_fp.cc" , "src/crypto/asn1/a_int.cc" , "src/crypto/asn1/a_mbstr.cc" , "src/crypto/asn1/a_object.cc" , "src/crypto/asn1/a_octet.cc" , "src/crypto/asn1/a_strex.cc" , "src/crypto/asn1/a_strnid.cc" , "src/crypto/asn1/a_time.cc" , "src/crypto/asn1/a_type.cc" , "src/crypto/asn1/a_utctm.cc" , "src/crypto/asn1/asn1_lib.cc" , "src/crypto/asn1/asn1_par.cc" , "src/crypto/asn1/asn_pack.cc" , "src/crypto/asn1/f_int.cc" , "src/crypto/asn1/f_string.cc" , "src/crypto/asn1/posix_time.cc" , "src/crypto/asn1/tasn_dec.cc" , "src/crypto/asn1/tasn_enc.cc" , "src/crypto/asn1/tasn_fre.cc" , "src/crypto/asn1/tasn_new.cc" , "src/crypto/asn1/tasn_typ.cc" , "src/crypto/asn1/tasn_utl.cc" , "src/crypto/base64/base64.cc" , "src/crypto/bio/bio.cc" , "src/crypto/bio/bio_mem.cc" , "src/crypto/bio/connect.cc" , "src/crypto/bio/errno.cc" , "src/crypto/bio/fd.cc" , "src/crypto/bio/file.cc" , "src/crypto/bio/hexdump.cc" , "src/crypto/bio/pair.cc" , "src/crypto/bio/printf.cc" , "src/crypto/bio/socket.cc" , "src/crypto/bio/socket_helper.cc" , "src/crypto/blake2/blake2.cc" , "src/crypto/bn_extra/bn_asn1.cc" , "src/crypto/bn_extra/convert.cc" , "src/crypto/buf/buf.cc" , "src/crypto/bytestring/asn1_compat.cc" , "src/crypto/bytestring/ber.cc" , "src/crypto/bytestring/cbb.cc" , "src/crypto/bytestring/cbs.cc" , "src/crypto/bytestring/unicode.cc" , "src/crypto/chacha/chacha.cc" , "src/crypto/cipher_extra/cipher_extra.cc" , "src/crypto/cipher_extra/derive_key.cc" , "src/crypto/cipher_extra/e_aesctrhmac.cc" , "src/crypto/cipher_extra/e_aesgcmsiv.cc" , "src/crypto/cipher_extra/e_chacha20poly1305.cc" , "src/crypto/cipher_extra/e_des.cc" , "src/crypto/cipher_extra/e_null.cc" , "src/crypto/cipher_extra/e_rc2.cc" , "src/crypto/cipher_extra/e_rc4.cc" , "src/crypto/cipher_extra/e_tls.cc" , "src/crypto/cipher_extra/tls_cbc.cc" , "src/crypto/conf/conf.cc" , "src/crypto/cpu_aarch64_apple.cc" , "src/crypto/cpu_aarch64_fuchsia.cc" , "src/crypto/cpu_aarch64_linux.cc" , "src/crypto/cpu_aarch64_openbsd.cc" , "src/crypto/cpu_aarch64_sysreg.cc" , "src/crypto/cpu_aarch64_win.cc" , "src/crypto/cpu_arm_freebsd.cc" , "src/crypto/cpu_arm_linux.cc" , "src/crypto/cpu_intel.cc" , "src/crypto/crypto.cc" , "src/crypto/curve25519/curve25519.cc" , "src/crypto/curve25519/curve25519_64_adx.cc" , "src/crypto/curve25519/spake25519.cc" , "src/crypto/des/des.cc" , "src/crypto/dh_extra/dh_asn1.cc" , "src/crypto/dh_extra/params.cc" , "src/crypto/digest_extra/digest_extra.cc" , "src/crypto/dsa/dsa.cc" , "src/crypto/dsa/dsa_asn1.cc" , "src/crypto/ec_extra/ec_asn1.cc" , "src/crypto/ec_extra/ec_derive.cc" , "src/crypto/ec_extra/hash_to_curve.cc" , "src/crypto/ecdh_extra/ecdh_extra.cc" , "src/crypto/ecdsa_extra/ecdsa_asn1.cc" , "src/crypto/engine/engine.cc" , "src/crypto/err/err.cc" , "src/crypto/evp/evp.cc" , "src/crypto/evp/evp_asn1.cc" , "src/crypto/evp/evp_ctx.cc" , "src/crypto/evp/p_dh.cc" , "src/crypto/evp/p_dh_asn1.cc" , "src/crypto/evp/p_dsa_asn1.cc" , "src/crypto/evp/p_ec.cc" , "src/crypto/evp/p_ec_asn1.cc" , "src/crypto/evp/p_ed25519.cc" , "src/crypto/evp/p_ed25519_asn1.cc" , "src/crypto/evp/p_hkdf.cc" , "src/crypto/evp/p_rsa.cc" , "src/crypto/evp/p_rsa_asn1.cc" , "src/crypto/evp/p_x25519.cc" , "src/crypto/evp/p_x25519_asn1.cc" , "src/crypto/evp/pbkdf.cc" , "src/crypto/evp/print.cc" , "src/crypto/evp/scrypt.cc" , "src/crypto/evp/sign.cc" , "src/crypto/ex_data.cc" , "src/crypto/fipsmodule/bcm.cc" , "src/crypto/fipsmodule/fips_shared_support.cc" , "src/crypto/hpke/hpke.cc" , "src/crypto/hrss/hrss.cc" , "src/crypto/keccak/keccak.cc" , "src/crypto/kyber/kyber.cc" , "src/crypto/lhash/lhash.cc" , "src/crypto/md4/md4.cc" , "src/crypto/md5/md5.cc" , "src/crypto/mem.cc" , "src/crypto/mldsa/mldsa.cc" , "src/crypto/mlkem/mlkem.cc" , "src/crypto/obj/obj.cc" , "src/crypto/obj/obj_xref.cc" , "src/crypto/pem/pem_all.cc" , "src/crypto/pem/pem_info.cc" , "src/crypto/pem/pem_lib.cc" , "src/crypto/pem/pem_oth.cc" , "src/crypto/pem/pem_pk8.cc" , "src/crypto/pem/pem_pkey.cc" , "src/crypto/pem/pem_x509.cc" , "src/crypto/pem/pem_xaux.cc" , "src/crypto/pkcs7/pkcs7.cc" , "src/crypto/pkcs7/pkcs7_x509.cc" , "src/crypto/pkcs8/p5_pbev2.cc" , "src/crypto/pkcs8/pkcs8.cc" , "src/crypto/pkcs8/pkcs8_x509.cc" , "src/crypto/poly1305/poly1305.cc" , "src/crypto/poly1305/poly1305_arm.cc" , "src/crypto/poly1305/poly1305_vec.cc" , "src/crypto/pool/pool.cc" , "src/crypto/rand_extra/deterministic.cc" , "src/crypto/rand_extra/fork_detect.cc" , "src/crypto/rand_extra/forkunsafe.cc" , "src/crypto/rand_extra/getentropy.cc" , "src/crypto/rand_extra/ios.cc" , "src/crypto/rand_extra/passive.cc" , "src/crypto/rand_extra/rand_extra.cc" , "src/crypto/rand_extra/trusty.cc" , "src/crypto/rand_extra/urandom.cc" , "src/crypto/rand_extra/windows.cc" , "src/crypto/rc4/rc4.cc" , "src/crypto/refcount.cc" , "src/crypto/rsa_extra/rsa_asn1.cc" , "src/crypto/rsa_extra/rsa_crypt.cc" , "src/crypto/rsa_extra/rsa_extra.cc" , "src/crypto/rsa_extra/rsa_print.cc" , "src/crypto/sha/sha1.cc" , "src/crypto/sha/sha256.cc" , "src/crypto/sha/sha512.cc" , "src/crypto/siphash/siphash.cc" , "src/crypto/slhdsa/fors.cc" , "src/crypto/slhdsa/merkle.cc" , "src/crypto/slhdsa/slhdsa.cc" , "src/crypto/slhdsa/thash.cc" , "src/crypto/slhdsa/wots.cc" , "src/crypto/stack/stack.cc" , "src/crypto/thread.cc" , "src/crypto/thread_none.cc" , "src/crypto/thread_pthread.cc" , "src/crypto/thread_win.cc" , "src/crypto/trust_token/pmbtoken.cc" , "src/crypto/trust_token/trust_token.cc" , "src/crypto/trust_token/voprf.cc" , "src/crypto/x509/a_digest.cc" , "src/crypto/x509/a_sign.cc" , "src/crypto/x509/a_verify.cc" , "src/crypto/x509/algorithm.cc" , "src/crypto/x509/asn1_gen.cc" , "src/crypto/x509/by_dir.cc" , "src/crypto/x509/by_file.cc" , "src/crypto/x509/i2d_pr.cc" , "src/crypto/x509/name_print.cc" , "src/crypto/x509/policy.cc" , "src/crypto/x509/rsa_pss.cc" , "src/crypto/x509/t_crl.cc" , "src/crypto/x509/t_req.cc" , "src/crypto/x509/t_x509.cc" , "src/crypto/x509/t_x509a.cc" , "src/crypto/x509/v3_akey.cc" , "src/crypto/x509/v3_akeya.cc" , "src/crypto/x509/v3_alt.cc" , "src/crypto/x509/v3_bcons.cc" , "src/crypto/x509/v3_bitst.cc" , "src/crypto/x509/v3_conf.cc" , "src/crypto/x509/v3_cpols.cc" , "src/crypto/x509/v3_crld.cc" , "src/crypto/x509/v3_enum.cc" , "src/crypto/x509/v3_extku.cc" , "src/crypto/x509/v3_genn.cc" , "src/crypto/x509/v3_ia5.cc" , "src/crypto/x509/v3_info.cc" , "src/crypto/x509/v3_int.cc" , "src/crypto/x509/v3_lib.cc" , "src/crypto/x509/v3_ncons.cc" , "src/crypto/x509/v3_ocsp.cc" , "src/crypto/x509/v3_pcons.cc" , "src/crypto/x509/v3_pmaps.cc" , "src/crypto/x509/v3_prn.cc" , "src/crypto/x509/v3_purp.cc" , "src/crypto/x509/v3_skey.cc" , "src/crypto/x509/v3_utl.cc" , "src/crypto/x509/x509.cc" , "src/crypto/x509/x509_att.cc" , "src/crypto/x509/x509_cmp.cc" , "src/crypto/x509/x509_d2.cc" , "src/crypto/x509/x509_def.cc" , "src/crypto/x509/x509_ext.cc" , "src/crypto/x509/x509_lu.cc" , "src/crypto/x509/x509_obj.cc" , "src/crypto/x509/x509_req.cc" , "src/crypto/x509/x509_set.cc" , "src/crypto/x509/x509_trs.cc" , "src/crypto/x509/x509_txt.cc" , "src/crypto/x509/x509_v3.cc" , "src/crypto/x509/x509_vfy.cc" , "src/crypto/x509/x509_vpm.cc" , "src/crypto/x509/x509cset.cc" , "src/crypto/x509/x509name.cc" , "src/crypto/x509/x509rset.cc" , "src/crypto/x509/x509spki.cc" , "src/crypto/x509/x_algor.cc" , "src/crypto/x509/x_all.cc" , "src/crypto/x509/x_attrib.cc" , "src/crypto/x509/x_crl.cc" , "src/crypto/x509/x_exten.cc" , "src/crypto/x509/x_name.cc" , "src/crypto/x509/x_pubkey.cc" , "src/crypto/x509/x_req.cc" , "src/crypto/x509/x_sig.cc" , "src/crypto/x509/x_spki.cc" , "src/crypto/x509/x_val.cc" , "src/crypto/x509/x_x509.cc" , "src/crypto/x509/x_x509a.cc" , "src/gen/crypto/err_data.cc" ] } , "crypto_internal_headers": { "type": "install" , "deps": [ "src/crypto/asn1/internal.h" , "src/crypto/bcm_support.h" , "src/crypto/bio/internal.h" , "src/crypto/bytestring/internal.h" , "src/crypto/chacha/internal.h" , "src/crypto/cipher_extra/internal.h" , "src/crypto/conf/internal.h" , "src/crypto/cpu_arm_linux.h" , "src/crypto/curve25519/curve25519_tables.h" , "src/crypto/curve25519/internal.h" , "src/crypto/des/internal.h" , "src/crypto/dsa/internal.h" , "src/crypto/ec_extra/internal.h" , "src/crypto/err/internal.h" , "src/crypto/evp/internal.h" , "src/crypto/fipsmodule/aes/internal.h" , "src/crypto/fipsmodule/bcm_interface.h" , "src/crypto/fipsmodule/bn/internal.h" , "src/crypto/fipsmodule/bn/rsaz_exp.h" , "src/crypto/fipsmodule/cipher/internal.h" , "src/crypto/fipsmodule/delocate.h" , "src/crypto/fipsmodule/dh/internal.h" , "src/crypto/fipsmodule/digest/internal.h" , "src/crypto/fipsmodule/digest/md32_common.h" , "src/crypto/fipsmodule/ec/builtin_curves.h" , "src/crypto/fipsmodule/ec/internal.h" , "src/crypto/fipsmodule/ec/p256-nistz-table.h" , "src/crypto/fipsmodule/ec/p256-nistz.h" , "src/crypto/fipsmodule/ec/p256_table.h" , "src/crypto/fipsmodule/ecdsa/internal.h" , "src/crypto/fipsmodule/modes/internal.h" , "src/crypto/fipsmodule/rand/internal.h" , "src/crypto/fipsmodule/rsa/internal.h" , "src/crypto/fipsmodule/service_indicator/internal.h" , "src/crypto/fipsmodule/sha/internal.h" , "src/crypto/fipsmodule/tls/internal.h" , "src/crypto/hrss/internal.h" , "src/crypto/internal.h" , "src/crypto/keccak/internal.h" , "src/crypto/kyber/internal.h" , "src/crypto/lhash/internal.h" , "src/crypto/md5/internal.h" , "src/crypto/mldsa/internal.h" , "src/crypto/mlkem/internal.h" , "src/crypto/obj/obj_dat.h" , "src/crypto/pkcs7/internal.h" , "src/crypto/pkcs8/internal.h" , "src/crypto/poly1305/internal.h" , "src/crypto/pool/internal.h" , "src/crypto/rand_extra/getrandom_fillin.h" , "src/crypto/rand_extra/sysrand_internal.h" , "src/crypto/rsa_extra/internal.h" , "src/crypto/slhdsa/address.h" , "src/crypto/slhdsa/fors.h" , "src/crypto/slhdsa/internal.h" , "src/crypto/slhdsa/merkle.h" , "src/crypto/slhdsa/params.h" , "src/crypto/slhdsa/thash.h" , "src/crypto/slhdsa/wots.h" , "src/crypto/trust_token/internal.h" , "src/crypto/x509/ext_dat.h" , "src/crypto/x509/internal.h" , "src/third_party/fiat/curve25519_32.h" , "src/third_party/fiat/curve25519_64.h" , "src/third_party/fiat/curve25519_64_adx.h" , "src/third_party/fiat/curve25519_64_msvc.h" , "src/third_party/fiat/p256_32.h" , "src/third_party/fiat/p256_64.h" , "src/third_party/fiat/p256_64_msvc.h" ] } , "crypto_sources_asm": { "type": "install" , "deps": [ "src/crypto/curve25519/asm/x25519-asm-arm.S" , "src/crypto/hrss/asm/poly_rq_mul.S" , "src/crypto/poly1305/poly1305_arm_asm.S" , "src/gen/crypto/aes128gcmsiv-x86_64-apple.S" , "src/gen/crypto/aes128gcmsiv-x86_64-linux.S" , "src/gen/crypto/chacha-armv4-linux.S" , "src/gen/crypto/chacha-armv8-apple.S" , "src/gen/crypto/chacha-armv8-linux.S" , "src/gen/crypto/chacha-armv8-win.S" , "src/gen/crypto/chacha-x86-apple.S" , "src/gen/crypto/chacha-x86-linux.S" , "src/gen/crypto/chacha-x86_64-apple.S" , "src/gen/crypto/chacha-x86_64-linux.S" , "src/gen/crypto/chacha20_poly1305_armv8-apple.S" , "src/gen/crypto/chacha20_poly1305_armv8-linux.S" , "src/gen/crypto/chacha20_poly1305_armv8-win.S" , "src/gen/crypto/chacha20_poly1305_x86_64-apple.S" , "src/gen/crypto/chacha20_poly1305_x86_64-linux.S" , "src/gen/crypto/md5-586-apple.S" , "src/gen/crypto/md5-586-linux.S" , "src/gen/crypto/md5-x86_64-apple.S" , "src/gen/crypto/md5-x86_64-linux.S" , "src/third_party/fiat/asm/fiat_curve25519_adx_mul.S" , "src/third_party/fiat/asm/fiat_curve25519_adx_square.S" ] } , "ssl_sources": { "type": "install" , "deps": [ "src/ssl/bio_ssl.cc" , "src/ssl/d1_both.cc" , "src/ssl/d1_lib.cc" , "src/ssl/d1_pkt.cc" , "src/ssl/d1_srtp.cc" , "src/ssl/dtls_method.cc" , "src/ssl/dtls_record.cc" , "src/ssl/encrypted_client_hello.cc" , "src/ssl/extensions.cc" , "src/ssl/handoff.cc" , "src/ssl/handshake.cc" , "src/ssl/handshake_client.cc" , "src/ssl/handshake_server.cc" , "src/ssl/s3_both.cc" , "src/ssl/s3_lib.cc" , "src/ssl/s3_pkt.cc" , "src/ssl/ssl_aead_ctx.cc" , "src/ssl/ssl_asn1.cc" , "src/ssl/ssl_buffer.cc" , "src/ssl/ssl_cert.cc" , "src/ssl/ssl_cipher.cc" , "src/ssl/ssl_credential.cc" , "src/ssl/ssl_file.cc" , "src/ssl/ssl_key_share.cc" , "src/ssl/ssl_lib.cc" , "src/ssl/ssl_privkey.cc" , "src/ssl/ssl_session.cc" , "src/ssl/ssl_stat.cc" , "src/ssl/ssl_transcript.cc" , "src/ssl/ssl_versions.cc" , "src/ssl/ssl_x509.cc" , "src/ssl/t1_enc.cc" , "src/ssl/tls13_both.cc" , "src/ssl/tls13_client.cc" , "src/ssl/tls13_enc.cc" , "src/ssl/tls13_server.cc" , "src/ssl/tls_method.cc" , "src/ssl/tls_record.cc" ] } , "ssl_internal_headers": {"type": "install", "deps": ["src/ssl/internal.h"]} } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.bzip2000066400000000000000000000052751516554100600237760ustar00rootroot00000000000000{ "libbz2": { "type": "export" , "target": "libbz2internal" , "doc": ["The BZip2 linkable library"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" ] } , "bzip2": { "type": "export" , "target": "bzip2internal" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" ] } , "bunzip2": {"type": "install", "files": {"bunzip2": "bzip2"}} , "bzcat": {"type": "install", "files": {"bzcat": "bzip2"}} , "bzip2recover": { "type": "export" , "target": "bzip2recoverinternal" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" ] } , "test": { "type": ["@", "rules", "shell/test", "script"] , "name": ["test"] , "test": ["test_script"] , "deps": ["bzip2", ["GLOB", null, "sample*"]] } , "test_script": { "type": "file_gen" , "name": "test.sh" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -e" , "./bzip2 -1 < sample1.ref > sample1.rb2" , "./bzip2 -2 < sample2.ref > sample2.rb2" , "./bzip2 -3 < sample3.ref > sample3.rb2" , "./bzip2 -d < sample1.bz2 > sample1.tst" , "./bzip2 -d < sample2.bz2 > sample2.tst" , "./bzip2 -ds < sample3.bz2 > sample3.tst" , "cmp sample1.bz2 sample1.rb2" , "cmp sample2.bz2 sample2.rb2" , "cmp sample3.bz2 sample3.rb2" , "cmp sample1.tst sample1.ref" , "cmp sample2.tst sample2.ref" , "cmp sample3.tst sample3.ref" ] } } , "libbz2internal": { "type": ["@", "rules", "CC", "library"] , "name": ["libbz2"] , "pure C": ["YES"] , "srcs": [ "blocksort.c" , "huffman.c" , "crctable.c" , "randtable.c" , "compress.c" , "decompress.c" , "bzlib.c" ] , "hdrs": ["bzlib.h"] , "private-hdrs": ["bzlib_private.h"] } , "bzip2internal": { "type": "configure" , "target": "bzip2internal (unconfigured)" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "bzip2internal (unconfigured)": { "type": ["@", "rules", "CC", "binary"] , "name": ["bzip2"] , "pure C": ["YES"] , "srcs": ["bzip2.c"] , "deps": ["libbz2"] } , "bzip2recoverinternal": { "type": "configure" , "target": "bzip2recoverinternal (unconfigured)" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "bzip2recoverinternal (unconfigured)": { "type": ["@", "rules", "CC", "binary"] , "name": ["bzip2recover"] , "pure C": ["YES"] , "srcs": ["bzip2recover.c"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.cares000066400000000000000000000004721516554100600240370ustar00rootroot00000000000000{ "ares": { "type": "export" , "target": ["src/lib", "ares_lib"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" , "BUILD_POSITION_INDEPENDENT" , "PKG_CONFIG_ARGS" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.catch2000066400000000000000000000032241516554100600241040ustar00rootroot00000000000000{ "catch2-config-header-blueprint": { "type": ["@", "rules", "CC/auto", "config_file"] , "input": [["./", "src/catch2", "catch_user_config.hpp.in"]] , "output": ["catch2", "catch_user_config.hpp"] , "magic_string": ["cmakedefine"] , "@only": ["true"] } , "catch2-config-header": { "type": "configure" , "target": "catch2-config-header-blueprint" , "config": { "type": "let*" , "bindings": [ [ "defines" , [ ["CATCH_CONFIG_DEFAULT_REPORTER", "console"] , ["CATCH_CONFIG_CONSOLE_WIDTH", "80"] ] ] ] , "body": {"type": "env", "vars": ["defines"]} } } , "catch2": { "type": "export" , "target": "catch2_internal" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "catch2-main": { "type": "export" , "target": "catch2-main_internal" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "catch2_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["Catch2"] , "hdrs": [["./", "src/catch2", "hdrs"], "catch2-config-header"] , "srcs": [["./", "src/catch2", "srcs"]] } , "catch2-main_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["Catch2Main"] , "hdrs": ["catch2-config-header"] , "srcs": [["./", "src/catch2", "main"]] , "deps": ["catch2"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.cli11000066400000000000000000000007241516554100600236530ustar00rootroot00000000000000{ "cli11": { "type": "export" , "target": "cli11 (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "cli11 (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["CLI11"] , "hdrs": [["./", "include/CLI", "cli11_headers"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.curl000066400000000000000000000537321516554100600237160ustar00rootroot00000000000000{ "curl": { "type": "export" , "target": "curl_config" , "doc": ["The Curl linkable library"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CURL_HIDDEN_SYMBOLS" , "USE_ZLIB" , "ENABLE_ARES" , "ENABLE_THREADED_RESOLVER" , "CURL_DISABLE_DICT" , "CURL_DISABLE_FILE" , "CURL_DISABLE_FORM_API" , "CURL_DISABLE_FTP" , "CURL_DISABLE_GOPHER" , "CURL_DISABLE_IMAP" , "CURL_DISABLE_LDAP" , "CURL_DISABLE_LDAPS" , "CURL_DISABLE_MQTT" , "CURL_DISABLE_POP3" , "CURL_DISABLE_RTSP" , "CURL_DISABLE_SMB" , "CURL_DISABLE_SMTP" , "CURL_DISABLE_TELNET" , "CURL_DISABLE_TFTP" , "HTTP_ONLY" , "CURL_DISABLE_ALTSVC" , "CURL_DISABLE_SRP" , "CURL_DISABLE_COOKIES" , "CURL_DISABLE_BASIC_AUTH" , "CURL_DISABLE_BEARER_AUTH" , "CURL_DISABLE_DIGEST_AUTH" , "CURL_DISABLE_KERBEROS_AUTH" , "CURL_DISABLE_NEGOTIATE_AUTH" , "CURL_DISABLE_AWS" , "CURL_DISABLE_NTLM" , "CURL_DISABLE_DOH" , "CURL_DISABLE_GETOPTIONS" , "CURL_DISABLE_HEADERS_API" , "CURL_DISABLE_BINDLOCAL" , "CURL_DISABLE_HSTS" , "CURL_DISABLE_MIME" , "CURL_DISABLE_NETRC" , "CURL_DISABLE_PARSEDATE" , "CURL_DISABLE_PROGRESS_METER" , "CURL_DISABLE_SHUFFLE_DNS" , "CURL_DISABLE_SOCKETPAIR" , "ENABLE_IPV6" , "CURL_ENABLE_SSL" , "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , "USE_NGHTTP2" , "USE_NGTCP2" , "USE_QUICHE" , "USE_MSH3" , "USE_LIBIDN2" , "USE_BROTLI" , "CURL_ZSTD" , "CURL_USE_LIBPSL" , "CURL_USE_LIBSSH2" , "CURL_USE_LIBSSH" , "CURL_USE_GSSAPI" , "ENABLE_UNIX_SOCKETS" , "CURL_CA_BUNDLE" , "CURL_CA_PATH" , "CURL_CA_FALLBACK" , "USE_GNU_STRERROR_R" , "HAVE_BORINGSSL" , "HAVE_AWSLC" , "HAVE_SSL_SET0_WBIO" , "HAVE_OPENSSL_SRP" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" , "PKG_CONFIG_ARGS" ] , "config_doc": { "CURL_HIDDEN_SYMBOLS": [ "Boolean. Default value: true." , "Hide all symbols not officially external." ] , "USE_ZLIB": [ "Boolean. Default value: true." , "Link against zlib library (system or open name)." ] , "ENABLE_ARES": [ "Boolean. Default value: false." , "Link against ares library (system or open name)." ] , "ENABLE_THREADED_RESOLVER": [ "Boolean. Enables AsynchDNS by threading if ~ENABLE_ARES==false~." , "If ~OS==\"windows\"~, uses win32 threads, otherwise pthreads." ] , "CURL_DISABLE_DICT": ["Boolean. Disable DICT."] , "CURL_DISABLE_FILE": ["Boolean. Disable FILE."] , "CURL_DISABLE_FORM_API": ["Boolean. Disable form API, if ~CURL_DISABLE_MIME==false~"] , "CURL_DISABLE_FTP": ["Boolean. Disable FTP."] , "CURL_DISABLE_GOPHER": ["Boolean. Disable GOPHER."] , "CURL_DISABLE_IMAP": ["Boolean. Disable IMAP."] , "CURL_DISABLE_LDAP": ["Boolean. Disable LDAP."] , "CURL_DISABLE_LDAPS": ["Boolean. Disable LDAPS."] , "CURL_DISABLE_MQTT": ["Boolean. Disable MQTT."] , "CURL_DISABLE_POP3": ["Boolean. Disable POP3."] , "CURL_DISABLE_RTSP": ["Boolean. Disable RTSP."] , "CURL_DISABLE_SMB": ["Boolean. Disable SMB."] , "CURL_DISABLE_SMTP": ["Boolean. Disable SMTP."] , "CURL_DISABLE_TELNET": ["Boolean. Disable TELNET."] , "CURL_DISABLE_TFTP": ["Boolean. Disable TFTP."] , "HTTP_ONLY": [ "true|null: Disables the following advanced options:" , "DICT, FILE, FTP, GOPHER, IMAP, LDAP, LDAPS," , "MQTT, POP3, RTSP, SMB, SMTP, TELNET, TFTP." , "false: Allows advanced options to be used, if selected." , "Currently these are unsupported." ] , "CURL_DISABLE_ALTSVC": ["Boolean. Disable alt-svc."] , "CURL_DISABLE_SRP": ["Boolean. Disable TLS-SRP support."] , "CURL_DISABLE_COOKIES": ["Boolean. Disable cookies."] , "CURL_DISABLE_BASIC_AUTH": ["Boolean. Disable Basic authentication."] , "CURL_DISABLE_BEARER_AUTH": ["Boolean. Disable Bearer authentication."] , "CURL_DISABLE_DIGEST_AUTH": ["Boolean. Disable Digest authentication."] , "CURL_DISABLE_KERBEROS_AUTH": ["Boolean. Disable Kerberos authentication."] , "CURL_DISABLE_NEGOTIATE_AUTH": ["Boolean. Disable negotiate authentication."] , "CURL_DISABLE_AWS": ["Boolean. Disable AWS-SIG4."] , "CURL_DISABLE_NTLM": ["Boolean. Disable NTLM support."] , "CURL_DISABLE_DOH": ["Boolean. Disable DNS-over-HTTPS."] , "CURL_DISABLE_GETOPTIONS": ["Boolean. Disable curl-easy-options API."] , "CURL_DISABLE_HEADERS_API": ["Boolean. Disable headers-api support."] , "CURL_DISABLE_BINDLOCAL": ["Boolean. Disable local binding support."] , "CURL_DISABLE_HSTS": ["Boolean. Disable HSTS support."] , "CURL_DISABLE_MIME": ["Boolean. Disable MIME support."] , "CURL_DISABLE_NETRC": ["Boolean. Disable netrc parser."] , "CURL_DISABLE_PARSEDATE": ["Boolean. Disable date parsing."] , "CURL_DISABLE_PROGRESS_METER": ["Boolean. Disable built-in progress meter."] , "CURL_DISABLE_SHUFFLE_DNS": ["Boolean. Disable shuffle DNS feature."] , "CURL_DISABLE_SOCKETPAIR": ["Boolean. Disable use of socketpair for curl_multi_poll."] , "ENABLE_IPV6": ["Boolean. Default value: true. Enable IPv6 support."] , "CURL_ENABLE_SSL": [ "Boolean. Default value: true." , "Link against OpenSSL/BoringSSL as default backend (system or open name)." ] , "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG": ["Boolean. Disable automatic loading of OpenSSL/BoringSSL config."] , "USE_NGHTTP2": [ "Boolean. Link against nghttp2 (system or open name)." , "Provides http/2 support." ] , "USE_NGTCP2": [ "Boolean. Link against ngtcp2 if OpenSSL/BoringSSL is used." , "Provides http/3 support." ] , "USE_QUICHE": [ "Boolean. Link against quiche (system or open name) if ~USE_NGTCP2==false~." , "Provides http/3 support." ] , "USE_MSH3": [ "Boolean. Link against msh3 (system or open name) if ~USE_NGTCP2==false~ and ~USE_QUICHE==false~." , "Provides http/3 support." ] , "USE_LIBIDN2": [ "Boolean. Link against libidn2 (system or open name) if ~OS==\"linux\"~" , "or against normaliz (system or open name) if ~OS==\"windows\"~." ] , "USE_BROTLI": ["Boolean. Link against brotli (system or open name)."] , "CURL_ZSTD": ["Boolean. Link against zstd (system or open name)."] , "CURL_USE_LIBPSL": [ "Boolean. Default value: true. Link against libpsl (system or open name)." ] , "CURL_USE_LIBSSH2": [ "Boolean. Default value: true. Link against libssh2 (system or open name)." , "Provides SSH support." ] , "CURL_USE_LIBSSH": [ "Boolean. Link against libssh (system or open name) if ~CURL_USE_LIBSSH2==false~." , "Provides SSH support." ] , "CURL_USE_GSSAPI": [ "Boolean. Link against gssapi (system or open name)." , "Currently only heimdal library is supported as provider of gssapi symbol" ] , "ENABLE_UNIX_SOCKETS": ["Boolean. Default value: true. Define unix domain sockets support"] , "CURL_CA_FALLBACK": ["Boolean. Use the CA store of the system or of the used TLS backend"] , "CURL_CA_BUNDLE": [ "\"auto\"|null: Enforces the use of the system or TLS backend CA certs," , "as if by ~CURL_CA_FALLBACK==true~." , "\"none\": Allow ~CURL_CA_FALLBACK~ value to decide CA certification." , ": Explicit path to the CA bundle to use. User must ensure validity. " ] , "CURL_CA_PATH": [ "\"auto\"|null: Enforces the use of the system or TLS backend CA certs," , "as if by ~CURL_CA_FALLBACK==true~." , "\"none\": Allow ~CURL_CA_FALLBACK~ value to decide CA certification." , ": Explicit path to the CA certificate to use. User must ensure validity." ] , "USE_GNU_STRERROR_R": [ "Boolean. Manually set whether to use GNU or POSIX version of ~strerror_r~ function." , "Only safe to set if _GNU_SOURCE is set no _POSIX_SOURCE overwrite exists." ] , "HAVE_BORINGSSL": [ "Boolean. OpenSSL is BoringSSL." , "If true, corresponding symbol detection logic is skipped." ] , "HAVE_AWSLC": [ "Boolean. OpenSSL is AWSLC." , "If true, corresponding symbol detection logic is skipped." ] , "HAVE_SSL_SET0_WBIO": ["Boolean. Set if ~SSL_set0_wbio~ present in OpenSSL/wolfSSL."] , "HAVE_OPENSSL_SRP": [ "Boolean. Set if ~SSL_CTX_set_srp_username~ present in OpenSSL/wolfSSL." ] , "HAVE_SSL_CTX_SET_QUIC_METHOD": [ "Boolean. `SSL_CTX_set_quic_method` present in OpenSSL." , "If true, corresponding symbol detection logic is skipped." ] , "HAVE_QUICHE_CONN_SET_QLOG_FD": [ "Boolean. `quiche_conn_set_qlog_fd` present in QUICHE." , "If true, corresponding symbol detection logic is skipped." ] } } , "curl_config": { "type": "configure" , "arguments_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "USE_SYSTEM_LIBS" , "CURL_HIDDEN_SYMBOLS" , "USE_ZLIB" , "ENABLE_ARES" , "ENABLE_THREADED_RESOLVER" , "CURL_DISABLE_DICT" , "CURL_DISABLE_FILE" , "CURL_DISABLE_FORM_API" , "CURL_DISABLE_FTP" , "CURL_DISABLE_GOPHER" , "CURL_DISABLE_IMAP" , "CURL_DISABLE_LDAP" , "CURL_DISABLE_LDAPS" , "CURL_DISABLE_MQTT" , "CURL_DISABLE_POP3" , "CURL_DISABLE_RTSP" , "CURL_DISABLE_SMB" , "CURL_DISABLE_SMTP" , "CURL_DISABLE_TELNET" , "CURL_DISABLE_TFTP" , "HTTP_ONLY" , "CURL_DISABLE_ALTSVC" , "CURL_DISABLE_SRP" , "CURL_DISABLE_COOKIES" , "CURL_DISABLE_BASIC_AUTH" , "CURL_DISABLE_BEARER_AUTH" , "CURL_DISABLE_DIGEST_AUTH" , "CURL_DISABLE_KERBEROS_AUTH" , "CURL_DISABLE_NEGOTIATE_AUTH" , "CURL_DISABLE_AWS" , "CURL_DISABLE_NTLM" , "CURL_DISABLE_DOH" , "CURL_DISABLE_GETOPTIONS" , "CURL_DISABLE_HEADERS_API" , "CURL_DISABLE_BINDLOCAL" , "CURL_DISABLE_HSTS" , "CURL_DISABLE_MIME" , "CURL_DISABLE_NETRC" , "CURL_DISABLE_PARSEDATE" , "CURL_DISABLE_PROGRESS_METER" , "CURL_DISABLE_SHUFFLE_DNS" , "CURL_DISABLE_SOCKETPAIR" , "ENABLE_IPV6" , "CURL_ENABLE_SSL" , "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , "USE_NGHTTP2" , "USE_NGTCP2" , "USE_QUICHE" , "USE_MSH3" , "USE_LIBIDN2" , "USE_BROTLI" , "CURL_ZSTD" , "CURL_USE_LIBPSL" , "CURL_USE_LIBSSH2" , "CURL_USE_LIBSSH" , "CURL_USE_GSSAPI" , "ENABLE_UNIX_SOCKETS" , "CURL_CA_BUNDLE" , "CURL_CA_PATH" , "CURL_CA_FALLBACK" , "USE_GNU_STRERROR_R" , "HAVE_BORINGSSL" , "HAVE_AWSLC" , "HAVE_SSL_SET0_WBIO" , "HAVE_OPENSSL_SRP" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" ] , "target": ["./", "lib", "libcurl"] , "config": { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] , [ "CURL_HIDDEN_SYMBOLS" , {"type": "var", "name": "CURL_HIDDEN_SYMBOLS", "default": true} ] , ["USE_ZLIB", {"type": "var", "name": "USE_ZLIB", "default": true}] , [ "ENABLE_ARES" , {"type": "var", "name": "ENABLE_ARES", "default": false} ] , [ "ENABLE_THREADED_RESOLVER" , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ARES"} , "then": false , "else": { "type": "var" , "name": "ENABLE_THREADED_RESOLVER" , "default": true } } ] , [ "CURL_DISABLE_DICT" , {"type": "var", "name": "CURL_DISABLE_DICT", "default": false} ] , [ "CURL_DISABLE_FILE" , {"type": "var", "name": "CURL_DISABLE_FILE", "default": false} ] , [ "CURL_DISABLE_FORM_API" , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_MIME"} , "then": true , "else": {"type": "var", "name": "CURL_DISABLE_FORM_API", "default": false} } ] , [ "CURL_DISABLE_FTP" , {"type": "var", "name": "CURL_DISABLE_FTP", "default": false} ] , [ "CURL_DISABLE_GOPHER" , {"type": "var", "name": "CURL_DISABLE_GOPHER", "default": false} ] , [ "CURL_DISABLE_IMAP" , {"type": "var", "name": "CURL_DISABLE_IMAP", "default": false} ] , [ "CURL_DISABLE_LDAP" , {"type": "var", "name": "CURL_DISABLE_LDAP", "default": false} ] , [ "CURL_DISABLE_LDAPS" , {"type": "var", "name": "CURL_DISABLE_LDAPS", "default": false} ] , [ "CURL_DISABLE_MQTT" , {"type": "var", "name": "CURL_DISABLE_MQTT", "default": false} ] , [ "CURL_DISABLE_POP3" , {"type": "var", "name": "CURL_DISABLE_POP3", "default": false} ] , [ "CURL_DISABLE_RTSP" , {"type": "var", "name": "CURL_DISABLE_RTSP", "default": false} ] , [ "CURL_DISABLE_SMB" , {"type": "var", "name": "CURL_DISABLE_SMB", "default": false} ] , [ "CURL_DISABLE_SMTP" , {"type": "var", "name": "CURL_DISABLE_SMTP", "default": false} ] , [ "CURL_DISABLE_TELNET" , {"type": "var", "name": "CURL_DISABLE_TELNET", "default": false} ] , [ "CURL_DISABLE_TFTP" , {"type": "var", "name": "CURL_DISABLE_TFTP", "default": false} ] , ["HTTP_ONLY", {"type": "var", "name": "HTTP_ONLY", "default": false}] , [ "CURL_DISABLE_ALTSVC" , {"type": "var", "name": "CURL_DISABLE_ALTSVC", "default": false} ] , [ "CURL_DISABLE_SRP" , {"type": "var", "name": "CURL_DISABLE_SRP", "default": false} ] , [ "CURL_DISABLE_COOKIES" , {"type": "var", "name": "CURL_DISABLE_COOKIES", "default": false} ] , [ "CURL_DISABLE_BASIC_AUTH" , {"type": "var", "name": "CURL_DISABLE_BASIC_AUTH", "default": false} ] , [ "CURL_DISABLE_BEARER_AUTH" , {"type": "var", "name": "CURL_DISABLE_BEARER_AUTH", "default": false} ] , [ "CURL_DISABLE_DIGEST_AUTH" , {"type": "var", "name": "CURL_DISABLE_DIGEST_AUTH", "default": false} ] , [ "CURL_DISABLE_KERBEROS_AUTH" , { "type": "var" , "name": "CURL_DISABLE_KERBEROS_AUTH" , "default": false } ] , [ "CURL_DISABLE_NEGOTIATE_AUTH" , { "type": "var" , "name": "CURL_DISABLE_NEGOTIATE_AUTH" , "default": false } ] , [ "CURL_DISABLE_AWS" , {"type": "var", "name": "CURL_DISABLE_AWS", "default": false} ] , [ "CURL_DISABLE_NTLM" , {"type": "var", "name": "CURL_DISABLE_NTLM", "default": false} ] , [ "CURL_DISABLE_DOH" , {"type": "var", "name": "CURL_DISABLE_DOH", "default": false} ] , [ "CURL_DISABLE_GETOPTIONS" , {"type": "var", "name": "CURL_DISABLE_GETOPTIONS", "default": false} ] , [ "CURL_DISABLE_HEADERS_API" , {"type": "var", "name": "CURL_DISABLE_HEADERS_API", "default": false} ] , [ "CURL_DISABLE_BINDLOCAL" , {"type": "var", "name": "CURL_DISABLE_BINDLOCAL", "default": false} ] , [ "CURL_DISABLE_HSTS" , {"type": "var", "name": "CURL_DISABLE_HSTS", "default": false} ] , [ "CURL_DISABLE_MIME" , {"type": "var", "name": "CURL_DISABLE_MIME", "default": false} ] , [ "CURL_DISABLE_NETRC" , {"type": "var", "name": "CURL_DISABLE_NETRC", "default": false} ] , [ "CURL_DISABLE_PARSEDATE" , {"type": "var", "name": "CURL_DISABLE_PARSEDATE", "default": false} ] , [ "CURL_DISABLE_PROGRESS_METER" , { "type": "var" , "name": "CURL_DISABLE_PROGRESS_METER" , "default": false } ] , [ "CURL_DISABLE_SHUFFLE_DNS" , {"type": "var", "name": "CURL_DISABLE_SHUFFLE_DNS", "default": false} ] , [ "CURL_DISABLE_SOCKETPAIR" , {"type": "var", "name": "CURL_DISABLE_SOCKETPAIR", "default": false} ] , [ "ENABLE_IPV6" , {"type": "var", "name": "ENABLE_IPV6", "default": true} ] , [ "CURL_ENABLE_SSL" , {"type": "var", "name": "CURL_ENABLE_SSL", "default": true} ] , [ "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , { "type": "var" , "name": "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , "default": false } ] , [ "USE_NGHTTP2" , {"type": "var", "name": "USE_NGHTTP2", "default": false} ] , ["USE_NGTCP2", {"type": "var", "name": "USE_NGTCP2", "default": false}] , ["USE_QUICHE", {"type": "var", "name": "USE_QUICHE", "default": false}] , ["USE_MSH3", {"type": "var", "name": "USE_MSH3", "default": false}] , [ "USE_LIBIDN2" , {"type": "var", "name": "USE_LIBIDN2", "default": false} ] , ["USE_BROTLI", {"type": "var", "name": "USE_BROTLI", "default": false}] , ["CURL_ZSTD", {"type": "var", "name": "CURL_ZSTD", "default": false}] , [ "CURL_USE_LIBPSL" , {"type": "var", "name": "CURL_USE_LIBPSL", "default": true} ] , [ "CURL_USE_LIBSSH2" , {"type": "var", "name": "CURL_USE_LIBSSH2", "default": true} ] , [ "CURL_USE_LIBSSH" , {"type": "var", "name": "CURL_USE_LIBSSH", "default": false} ] , [ "CURL_USE_GSSAPI" , {"type": "var", "name": "CURL_USE_GSSAPI", "default": false} ] , [ "ENABLE_UNIX_SOCKETS" , {"type": "var", "name": "ENABLE_UNIX_SOCKETS", "default": true} ] , [ "CURL_CA_BUNDLE" , {"type": "var", "name": "CURL_CA_BUNDLE", "default": "auto"} ] , [ "CURL_CA_PATH" , {"type": "var", "name": "CURL_CA_PATH", "default": "auto"} ] , [ "CURL_CA_FALLBACK" , { "type": "case*" , "expr": {"type": "var", "name": "CURL_CA_FALLBACK", "default": false} , "case": [ [ false , { "type": "if" , "cond": { "type": "or" , "$1": [ { "type": "==" , "$1": {"type": "var", "name": "CURL_CA_BUNDLE"} , "$2": "auto" } , { "type": "==" , "$1": {"type": "var", "name": "CURL_CA_PATH"} , "$2": "auto" } ] } , "then": true , "else": false } ] ] , "default": {"type": "var", "name": "CURL_CA_FALLBACK"} } ] , [ "USE_GNU_STRERROR_R" , {"type": "var", "name": "USE_GNU_STRERROR_R", "default": false} ] , [ "HAVE_BORINGSSL" , {"type": "var", "name": "HAVE_BORINGSSL", "default": false} ] , ["HAVE_AWSLC", {"type": "var", "name": "HAVE_AWSLC", "default": false}] , [ "HAVE_SSL_SET0_WBIO" , {"type": "var", "name": "HAVE_SSL_SET0_WBIO", "default": false} ] , [ "HAVE_OPENSSL_SRP" , {"type": "var", "name": "HAVE_OPENSSL_SRP", "default": false} ] , [ "HAVE_SSL_CTX_SET_QUIC_METHOD" , { "type": "var" , "name": "HAVE_SSL_CTX_SET_QUIC_METHOD" , "default": false } ] , [ "HAVE_QUICHE_CONN_SET_QLOG_FD" , { "type": "var" , "name": "HAVE_QUICHE_CONN_SET_QLOG_FD" , "default": false } ] ] , "body": { "type": "env" , "vars": [ "USE_SYSTEM_LIBS" , "CURL_HIDDEN_SYMBOLS" , "ENABLE_ARES" , "ENABLE_THREADED_RESOLVER" , "CURL_DISABLE_DICT" , "CURL_DISABLE_FILE" , "CURL_DISABLE_FORM_API" , "CURL_DISABLE_FTP" , "CURL_DISABLE_GOPHER" , "CURL_DISABLE_IMAP" , "CURL_DISABLE_LDAP" , "CURL_DISABLE_LDAPS" , "CURL_DISABLE_MQTT" , "CURL_DISABLE_POP3" , "CURL_DISABLE_RTSP" , "CURL_DISABLE_SMB" , "CURL_DISABLE_SMTP" , "CURL_DISABLE_TELNET" , "CURL_DISABLE_TFTP" , "HTTP_ONLY" , "CURL_DISABLE_ALTSVC" , "CURL_DISABLE_SRP" , "CURL_DISABLE_COOKIES" , "CURL_DISABLE_BASIC_AUTH" , "CURL_DISABLE_BEARER_AUTH" , "CURL_DISABLE_DIGEST_AUTH" , "CURL_DISABLE_KERBEROS_AUTH" , "CURL_DISABLE_NEGOTIATE_AUTH" , "CURL_DISABLE_AWS" , "CURL_DISABLE_NTLM" , "CURL_DISABLE_DOH" , "CURL_DISABLE_GETOPTIONS" , "CURL_DISABLE_HEADERS_API" , "CURL_DISABLE_BINDLOCAL" , "CURL_DISABLE_HSTS" , "CURL_DISABLE_MIME" , "CURL_DISABLE_NETRC" , "CURL_DISABLE_PARSEDATE" , "CURL_DISABLE_PROGRESS_METER" , "CURL_DISABLE_SHUFFLE_DNS" , "CURL_DISABLE_SOCKETPAIR" , "ENABLE_IPV6" , "CURL_ENABLE_SSL" , "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , "USE_NGHTTP2" , "USE_NGTCP2" , "USE_QUICHE" , "USE_MSH3" , "USE_LIBIDN2" , "USE_BROTLI" , "CURL_ZSTD" , "CURL_USE_LIBPSL" , "CURL_USE_LIBSSH2" , "CURL_USE_LIBSSH" , "CURL_USE_GSSAPI" , "ENABLE_UNIX_SOCKETS" , "CURL_CA_BUNDLE" , "CURL_CA_PATH" , "CURL_CA_FALLBACK" , "USE_GNU_STRERROR_R" , "HAVE_BORINGSSL" , "HAVE_AWSLC" , "HAVE_SSL_SET0_WBIO" , "HAVE_OPENSSL_SRP" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.distfiles000066400000000000000000000002401516554100600247210ustar00rootroot00000000000000{ "distdir": {"type": "export", "target": "distdir (unexported)"} , "distdir (unexported)": {"type": "install", "dirs": [[["TREE", null, "."], "distdir"]]} } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.fmt000066400000000000000000000006351516554100600235310ustar00rootroot00000000000000{ "fmt-lib": { "type": ["@", "rules", "CC", "library"] , "name": ["fmt"] , "srcs": ["src/format.cc", "src/os.cc"] , "hdrs": [["./", "include/fmt", "hdrs"]] } , "fmt": { "type": "export" , "target": "fmt-lib" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.git2000066400000000000000000000173031516554100600236100ustar00rootroot00000000000000{ "git2": { "type": "export" , "target": "libgit2configured" , "doc": ["The Git linkable library"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "USE_SYSTEM_LIBS" , "DEBUG_POOL" , "USE_THREADS" , "REGEX_BACKEND" , "WINHTTP" , "USE_ICONV" , "USE_NSEC" , "USE_SSH" , "USE_NTLMCLIENT" , "USE_GSSAPI" , "USE_SHA1" , "USE_SHA256" , "USE_HTTPS" , "USE_HTTP_PARSER" , "USE_BUNDLED_ZLIB" , "PKG_CONFIG_ARGS" ] , "config_doc": { "REGEX_BACKEND": [ "\"regcomp_l\": compile against regcomp_l (always from system)" , "\"regcomp\": compile against regcomp (always from system)" , "\"pcre\": link against libpcre (from system or open name)" , "\"pcre2\": link against libpcre2 (from system or open name)" , "\"builtin\"|null: compile and link bundled pcre (./deps/pcre)" ] , "USE_GSSAPI": [ "true: use \"GSS.framework\" for ~OS==\"darwin\"~ or else \"gssapi\"" , "false: Disable GSS" , "\"GSS.framework\": Link against GSS framework (system or open name)" , "\"gssapi\": Link against libgssapi (system or open name)" ] , "USE_HTTPS": [ "true: use \"WinHTTP\" if ~WINHTTP~ and ~OS==\"windows\"~" , "\"SecureTransport\": link against Security framework (from system or open name)" , "\"WinHTTP\": link against Windows' libwinhttp (always from system)" , "\"OpenSSL\": link against OpenSSL (from system or open name)" , "\"mbedTLS\": link against mbedTLS (from system or open name)" , "false|null: Disable HTTPS" ] , "USE_HTTP_PARSER": [ "\"system\": link against libhttp_parser (from system or open name)" , ": compile and link bundled http_parser (./deps/http-parser)" ] , "USE_SHA1": [ "true: use \"CollisionDetection\"" , "\"CollisionDetection\": build with shipped SHA1DC implementation" , "\"OpenSSL\": link against OpenSSL compat library (from system or open name)" , "\"mbedTLS\": link against mbedTLS (from system or open name)" , "\"Win32\": link against Windows' SHA1 implementation (always from system)" , "\"CommonCrypto\": build with shipped common_crypto implementation" , "\"HTTPS\": inherit from ~USE_HTTPS~, or fall back to \"CollisionDetection\"" ] , "USE_SHA256": [ "true: use \"Builtin\"" , "\"Builtin\": build with shipped RFC 6234 implementation" , "\"OpenSSL\": link against OpenSSL compat library (from system or open name)" , "\"mbedTLS\": link against mbedTLS (from system or open name)" , "\"Win32\": link against Windows' SHA1 implementation (always from system)" , "\"CommonCrypto\": build with shipped common_crypto implementation" , "\"HTTPS\": inherit from ~USE_HTTPS~, or fall back to \"Builtin\"" ] , "WINHTTP": ["boolean. use \"WinHTTP\" if \"USE_HTTPS\" is true"] } } , "libgit2configured": { "type": "configure" , "arguments_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "USE_SYSTEM_LIBS" , "DEBUG_POOL" , "USE_THREADS" , "REGEX_BACKEND" , "WINHTTP" , "USE_ICONV" , "USE_NSEC" , "USE_SSH" , "USE_NTLMCLIENT" , "USE_GSSAPI" , "USE_SHA1" , "USE_SHA256" , "USE_HTTPS" , "USE_HTTP_PARSER" , "USE_BUNDLED_ZLIB" ] , "target": ["./", "src", "libgit2"] , "config": { "type": "let*" , "bindings": [ [ "check that architecture is set" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] , [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "USE_THREADS" , {"type": "var", "name": "USE_THREADS", "default": true} ] , ["USE_SSH", {"type": "var", "name": "USE_SSH", "default": true}] , ["USE_GSSAPI", {"type": "var", "name": "USE_GSSAPI", "default": true}] , ["USE_SHA1", {"type": "var", "name": "USE_SHA1", "default": true}] , ["USE_SHA256", {"type": "var", "name": "USE_SHA256", "default": true}] , ["USE_HTTPS", {"type": "var", "name": "USE_HTTPS", "default": true}] , [ "WINHTTP" , { "type": "var" , "name": "WINHTTP" , "default": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } } ] , [ "USE_HTTPS" , { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ [ true , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "if" , "cond": {"type": "var", "name": "WINHTTP"} , "then": "WinHTTP" , "else": true } ] ] , "default": true } ] ] , "default": {"type": "var", "name": "USE_HTTPS"} } ] , [ "USE_GSSAPI" , { "type": "case*" , "expr": {"type": "var", "name": "USE_GSSAPI"} , "case": [ [ true , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["darwin", "GSS.framework"]] , "default": "gssapi" } ] ] , "default": {"type": "var", "name": "USE_GSSAPI"} } ] , [ "USE_SHA1" , { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA1"} , "case": [ [true, "CollisionDetection"] , [ "HTTPS" , { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ ["SecureTransport", "CommonCrypto"] , ["WinHTTP", "Win32"] , [false, "CollisionDetection"] , [null, "CollisionDetection"] ] , "default": {"type": "var", "name": "USE_HTTPS"} } ] ] , "default": {"type": "var", "name": "USE_SHA1"} } ] , [ "USE_SHA256" , { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA256"} , "case": [ [true, "Builtin"] , [ "HTTPS" , { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ ["SecureTransport", "CommonCrypto"] , ["WinHTTP", "Win32"] , [false, "Builtin"] , [null, "Builtin"] ] , "default": {"type": "var", "name": "USE_HTTPS"} } ] ] , "default": {"type": "var", "name": "USE_SHA256"} } ] ] , "body": { "type": "env" , "vars": [ "USE_SYSTEM_LIBS" , "DEBUG_POOL" , "USE_THREADS" , "REGEX_BACKEND" , "USE_ICONV" , "USE_NSEC" , "USE_SSH" , "USE_NTLMCLIENT" , "USE_GSSAPI" , "USE_SHA1" , "USE_SHA256" , "USE_HTTPS" , "USE_HTTP_PARSER" , "USE_BUNDLED_ZLIB" ] } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.google_apis000066400000000000000000000067441516554100600252420ustar00rootroot00000000000000{ "google_api_http_proto": {"type": "export", "target": "google_api_http_proto (unexported)"} , "google_api_http_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_http_proto"] , "srcs": ["google/api/http.proto"] } , "google_api_httpbody_proto": {"type": "export", "target": "google_api_httpbody_proto (unexported)"} , "google_api_httpbody_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_httpbody_proto"] , "srcs": ["google/api/httpbody.proto"] } , "google_api_annotations_proto": {"type": "export", "target": "google_api_annotations_proto (unexported)"} , "google_api_annotations_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_annotations_proto"] , "srcs": ["google/api/annotations.proto"] , "deps": ["google_api_http_proto"] } , "google_api_client_proto": {"type": "export", "target": "google_api_client_proto (unexported)"} , "google_api_client_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_client_proto"] , "srcs": ["google/api/client.proto"] , "deps": ["google_api_launch_stage_proto"] } , "google_api_launch_stage_proto": {"type": "export", "target": "google_api_launch_stage_proto (unexported)"} , "google_api_launch_stage_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_launch_stage_proto"] , "srcs": ["google/api/launch_stage.proto"] } , "google_api_expr_v1alpha1_checked_proto": { "type": "export" , "target": "google_api_expr_v1alpha1_checked_proto (unexported)" } , "google_api_expr_v1alpha1_checked_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_expr_v1alpha1_checked_proto"] , "srcs": ["google/api/expr/v1alpha1/checked.proto"] , "deps": ["google_api_expr_v1alpha1_syntax_proto"] } , "google_api_expr_v1alpha1_syntax_proto": { "type": "export" , "target": "google_api_expr_v1alpha1_syntax_proto (unexported)" } , "google_api_expr_v1alpha1_syntax_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_api_expr_v1alpha1_syntax_proto"] , "srcs": ["google/api/expr/v1alpha1/syntax.proto"] } , "google_bytestream_proto": { "type": "export" , "target": "google_bytestream_proto (unexported)" , "flexible_config": ["PATCH", "ENV", "ARCH", "OS", "TOOLCHAIN_CONFIG"] } , "google_bytestream_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_bytestream_proto"] , "service": ["yes"] , "srcs": ["google/bytestream/bytestream.proto"] , "deps": ["google_api_annotations_proto"] } , "google_rpc_status_proto": {"type": "export", "target": "google_rpc_status_proto (unexported)"} , "google_rpc_status_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_rpc_status_proto"] , "srcs": ["google/rpc/status.proto"] } , "google/rpc/status.proto": {"type": "export", "target": ["FILE", null, "google/rpc/status.proto"]} , "google_longrunning_operations_proto": { "type": "export" , "target": "google_longrunning_operations_proto (unexported)" } , "google_longrunning_operations_proto (unexported)": { "type": ["@", "rules", "proto", "library"] , "name": ["google_longrunning_operations_proto"] , "service": ["yes"] , "srcs": ["google/longrunning/operations.proto"] , "deps": [ "google_api_annotations_proto" , "google_api_client_proto" , "google_rpc_status_proto" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.grpc000066400000000000000000003613741516554100600237100ustar00rootroot00000000000000{ "grpcxx_sources": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "src/cpp/client/call_credentials.cc" , "src/cpp/client/channel_cc.cc" , "src/cpp/client/channel_credentials.cc" , "src/cpp/client/client_callback.cc" , "src/cpp/client/client_context.cc" , "src/cpp/client/client_interceptor.cc" , "src/cpp/client/client_stats_interceptor.cc" , "src/cpp/client/create_channel.cc" , "src/cpp/client/create_channel_internal.cc" , "src/cpp/client/create_channel_posix.cc" , "src/cpp/common/alarm.cc" , "src/cpp/common/channel_arguments.cc" , "src/cpp/common/completion_queue_cc.cc" , "src/cpp/common/resource_quota_cc.cc" , "src/cpp/common/rpc_method.cc" , "src/cpp/common/version_cc.cc" , "src/cpp/common/validate_service_config.cc" , "src/cpp/server/async_generic_service.cc" , "src/cpp/server/channel_argument_option.cc" , "src/cpp/server/create_default_thread_pool.cc" , "src/cpp/server/external_connection_acceptor_impl.cc" , "src/cpp/server/health/default_health_check_service.cc" , "src/cpp/server/health/health_check_service.cc" , "src/cpp/server/health/health_check_service_server_builder_option.cc" , "src/cpp/server/server_builder.cc" , "src/cpp/server/server_callback.cc" , "src/cpp/server/server_cc.cc" , "src/cpp/server/server_context.cc" , "src/cpp/server/server_credentials.cc" , "src/cpp/server/server_posix.cc" , "src/cpp/thread_manager/thread_manager.cc" , "src/cpp/util/byte_buffer_cc.cc" , "src/cpp/util/string_ref.cc" , "src/cpp/util/time_cc.cc" ] } , "grpcxx_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "src/cpp/client/create_channel_internal.h" , "src/cpp/client/client_stats_interceptor.h" , "src/cpp/server/dynamic_thread_pool.h" , "src/cpp/server/external_connection_acceptor_impl.h" , "src/cpp/server/health/default_health_check_service.h" , "src/cpp/server/thread_pool_interface.h" , "src/cpp/thread_manager/thread_manager.h" ] } , "grpc_unsecure": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_unsecure"] , "srcs": [ "src/core/lib/surface/init.cc" , "src/core/plugin_registry/grpc_plugin_registry.cc" , "src/core/plugin_registry/grpc_plugin_registry_noextra.cc" ] , "hdrs": [["include/grpc", "grpc_public_headers"]] , "deps": [ "channel_arg_names" , "channel_stack_builder" , "config" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_client_channel" , "grpc_common" , "grpc_core_credentials_header" , "grpc_http_filters" , "grpc_security_base" , "grpc_trace" , "http_connect_handshaker" , "iomgr_timer" , "server" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["src/core", "channel_args"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "client_channel_backup_poller"] , ["src/core", "default_event_engine"] , ["src/core", "endpoint_info_handshaker"] , ["src/core", "experiments"] , ["src/core", "forkable"] , ["src/core", "grpc_authorization_base"] , ["src/core", "http_proxy_mapper"] , ["src/core", "init_internally"] , ["src/core", "posix_event_engine_timer_manager"] , ["src/core", "server_call_tracer_filter"] , ["src/core", "service_config_channel_arg_filter"] , ["src/core", "slice"] , ["src/core", "tcp_connect_handshaker"] , ["third_party/address_sorting", "address_sorting"] ] } , "grpc": { "type": "export" , "target": "grpc_internal" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "PKG_CONFIG_ARGS" ] } , "grpc_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc"] , "srcs": [ "src/core/lib/surface/init.cc" , "src/core/plugin_registry/grpc_plugin_registry.cc" , "src/core/plugin_registry/grpc_plugin_registry_extra.cc" ] , "hdrs": [["include/grpc", "grpc_public_headers"]] , "deps": [ "channel_arg_names" , "channel_stack_builder" , "config" , "exec_ctx" , "gpr" , "grpc_alts_credentials" , "grpc_base" , "grpc_client_channel" , "grpc_common" , "grpc_core_credentials_header" , "grpc_credentials_util" , "grpc_http_filters" , "grpc_jwt_credentials" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_trace" , "http_connect_handshaker" , "httpcli" , "iomgr_timer" , "promise" , "ref_counted_ptr" , "server" , "sockaddr_utils" , "tsi_base" , "uri" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["src/core", "channel_args"] , ["src/core", "channel_creds_registry_init"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "client_channel_backup_poller"] , ["src/core", "default_event_engine"] , ["src/core", "endpoint_info_handshaker"] , ["src/core", "experiments"] , ["src/core", "forkable"] , ["src/core", "grpc_authorization_base"] , ["src/core", "grpc_external_account_credentials"] , ["src/core", "grpc_fake_credentials"] , ["src/core", "grpc_google_default_credentials"] , ["src/core", "grpc_iam_credentials"] , ["src/core", "grpc_insecure_credentials"] , ["src/core", "grpc_lb_policy_cds"] , ["src/core", "grpc_lb_policy_ring_hash"] , ["src/core", "grpc_lb_policy_xds_cluster_impl"] , ["src/core", "grpc_lb_policy_xds_cluster_manager"] , ["src/core", "grpc_lb_policy_xds_override_host"] , ["src/core", "grpc_lb_policy_xds_wrr_locality"] , ["src/core", "grpc_local_credentials"] , ["src/core", "grpc_oauth2_credentials"] , ["src/core", "grpc_resolver_c2p"] , ["src/core", "grpc_resolver_xds"] , ["src/core", "grpc_ssl_credentials"] , ["src/core", "grpc_stateful_session_filter"] , ["src/core", "grpc_tls_credentials"] , ["src/core", "grpc_transport_chttp2_alpn"] , ["src/core", "grpc_xds_server_config_fetcher"] , ["src/core", "http_proxy_mapper"] , ["src/core", "httpcli_ssl_credentials"] , ["src/core", "init_internally"] , ["src/core", "json"] , ["src/core", "posix_event_engine_timer_manager"] , ["src/core", "ref_counted"] , ["src/core", "server_call_tracer_filter"] , ["src/core", "service_config_channel_arg_filter"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "tcp_connect_handshaker"] , ["src/core", "useful"] , ["src/core", "xds_http_proxy_mapper"] , ["third_party/address_sorting", "address_sorting"] ] } , "gpr": { "type": "export" , "target": "gpr_internal" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "PKG_CONFIG_ARGS" ] } , "gpr_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr"] , "srcs": [ "src/core/util/alloc.cc" , "src/core/util/crash.cc" , "src/core/util/fork.cc" , "src/core/util/gpr_time.cc" , "src/core/util/host_port.cc" , "src/core/util/iphone/cpu.cc" , "src/core/util/linux/cpu.cc" , "src/core/util/log.cc" , "src/core/util/mpscq.cc" , "src/core/util/msys/tmpfile.cc" , "src/core/util/posix/cpu.cc" , "src/core/util/posix/stat.cc" , "src/core/util/posix/string.cc" , "src/core/util/posix/sync.cc" , "src/core/util/posix/thd.cc" , "src/core/util/posix/time.cc" , "src/core/util/posix/tmpfile.cc" , "src/core/util/string.cc" , "src/core/util/sync.cc" , "src/core/util/sync_abseil.cc" , "src/core/util/time_precise.cc" , "src/core/util/time_util.cc" , "src/core/util/windows/cpu.cc" , "src/core/util/windows/stat.cc" , "src/core/util/windows/string.cc" , "src/core/util/windows/string_util.cc" , "src/core/util/windows/sync.cc" , "src/core/util/windows/thd.cc" , "src/core/util/windows/time.cc" , "src/core/util/windows/tmpfile.cc" ] , "hdrs": [ "src/core/util/alloc.h" , "src/core/util/crash.h" , "src/core/util/fork.h" , "src/core/util/host_port.h" , "src/core/util/memory.h" , "src/core/util/mpscq.h" , "src/core/util/stat.h" , "src/core/util/string.h" , "src/core/util/sync.h" , "src/core/util/thd.h" , "src/core/util/time_precise.h" , "src/core/util/time_util.h" , "src/core/util/tmpfile.h" , ["include/grpc", "gpr_public_headers"] ] , "deps": [ "config_vars" , "debug_location" , ["@", "absl", "absl/base", "base"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/base", "log_severity"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "globals"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["src/core", "construct_destruct"] , ["src/core", "env"] , ["src/core", "event_engine_thread_local"] , ["src/core", "examine_stack"] , ["src/core", "gpr_atm"] , ["src/core", "no_destruct"] , ["src/core", "strerror"] , ["src/core", "tchar"] , ["src/core", "useful"] ] } , "gpr_public_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr_public_hdrs"] , "hdrs": [["include/grpc", "gpr_public_headers"]] , "deps": [ ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] ] } , "cpp_impl_of": { "type": ["@", "rules", "CC", "library"] , "name": ["cpp_impl_of"] , "hdrs": ["src/core/util/cpp_impl_of.h"] } , "grpc_common": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_common"] , "deps": [ "census" , "grpc_base" , "grpc_resolver_dns_ares" , "grpc_resolver_fake" , ["src/core", "grpc_backend_metric_filter"] , ["src/core", "grpc_channel_idle_filter"] , ["src/core", "grpc_client_authority_filter"] , ["src/core", "grpc_fault_injection_filter"] , ["src/core", "grpc_lb_policy_grpclb"] , ["src/core", "grpc_lb_policy_outlier_detection"] , ["src/core", "grpc_lb_policy_pick_first"] , ["src/core", "grpc_lb_policy_priority"] , ["src/core", "grpc_lb_policy_rls"] , ["src/core", "grpc_lb_policy_round_robin"] , ["src/core", "grpc_lb_policy_weighted_round_robin"] , ["src/core", "grpc_lb_policy_weighted_target"] , ["src/core", "grpc_message_size_filter"] , ["src/core", "grpc_resolver_dns_native"] , ["src/core", "grpc_resolver_dns_plugin"] , ["src/core", "grpc_resolver_sockaddr"] , ["src/core", "grpc_transport_chttp2_client_connector"] , ["src/core", "grpc_transport_chttp2_server"] , ["src/core", "grpc_transport_inproc"] ] } , "grpc_public_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_public_hdrs"] , "hdrs": [ ["include/grpc", "grpc_public_headers"] , ["include/grpc", "grpc_public_event_engine_headers"] ] , "deps": [ "channel_arg_names" , "gpr_public_hdrs" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] ] } , "grpc++_public_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_public_hdrs"] , "hdrs": [ ["include/grpc++", "grpc++_public_headers"] , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "global_callback_hook" , "grpc_public_hdrs" , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "gpr_atm"] ] } , "channel_arg_names": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_arg_names"] , "hdrs": [["include/grpc", "channel_arg_names_headers"]] } , "grpc_slice": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_slice"] , "hdrs": ["include/grpc/slice.h", "include/grpc/slice_buffer.h"] , "deps": [["src/core", "slice"], ["src/core", "slice_buffer"]] } , "grpc++": { "type": "export" , "target": "grpc++_internal" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "PKG_CONFIG_ARGS" ] } , "grpc++_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++"] , "hdrs": [ "src/cpp/client/secure_credentials.h" , "src/cpp/common/secure_auth_context.h" , "src/cpp/server/secure_server_credentials.h" , ["include/grpc++", "grpc++_public_headers"] , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "global_callback_hook" , "grpc++_base" , "grpc++_xds_client" , "grpc++_xds_server" , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "cord"] , ["src/core", "gpr_atm"] , ["src/core", "slice"] ] } , "grpc_cronet_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_cronet_hdrs"] , "hdrs": ["include/grpc/grpc_cronet.h"] , "deps": ["gpr_public_hdrs", "grpc_base"] } , "grpc++_cronet_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_cronet_credentials"] , "srcs": ["src/cpp/client/cronet_credentials.cc"] , "hdrs": ["include/grpcpp/security/cronet_credentials.h"] , "deps": ["grpc++_base", "grpc_cronet_hdrs", "grpc_public_hdrs"] } , "grpc_authorization_provider": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_authorization_provider"] , "srcs": [ "src/core/lib/security/authorization/grpc_authorization_policy_provider.cc" , "src/core/lib/security/authorization/rbac_translator.cc" ] , "hdrs": [ "src/core/lib/security/authorization/grpc_authorization_policy_provider.h" , "src/core/lib/security/authorization/rbac_translator.h" ] , "deps": [ "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , "ref_counted_ptr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "error"] , ["src/core", "grpc_audit_logging"] , ["src/core", "grpc_authorization_base"] , ["src/core", "grpc_matchers"] , ["src/core", "grpc_rbac_engine"] , ["src/core", "json"] , ["src/core", "json_reader"] , ["src/core", "load_file"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "status_helper"] , ["src/core", "useful"] ] } , "grpc++_authorization_provider": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_authorization_provider"] , "srcs": ["src/cpp/server/authorization_policy_provider.cc"] , "hdrs": ["include/grpcpp/security/authorization_policy_provider.h"] , "deps": [ "gpr" , "grpc++" , "grpc++_public_hdrs" , "grpc_authorization_provider" , "grpc_public_hdrs" ] } , "grpc_cel_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_cel_engine"] , "srcs": ["src/core/lib/security/authorization/cel_authorization_engine.cc"] , "hdrs": ["src/core/lib/security/authorization/cel_authorization_engine.h"] , "deps": [ "gpr" , "grpc_mock_cel" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["src/core", "grpc_authorization_base"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] , ["third_party/upb", "message"] ] } , "grpc++_xds_client": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_xds_client"] , "srcs": ["src/cpp/client/xds_credentials.cc"] , "hdrs": ["src/cpp/client/secure_credentials.h"] , "deps": [ "exec_ctx" , "gpr" , "grpc" , "grpc++_base" , "grpc_base" , "grpc_public_hdrs" , "grpc_security_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] ] } , "grpc++_xds_server": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_xds_server"] , "srcs": [ "src/cpp/server/xds_server_builder.cc" , "src/cpp/server/xds_server_credentials.cc" ] , "hdrs": [ "src/cpp/server/secure_server_credentials.h" , ["include/grpcpp", "grpcpp_xds_server_builder_headers"] ] , "deps": [ "channel_arg_names" , "gpr" , "grpc" , "grpc++_base" , ["@", "absl", "absl/log", "check"] , ["src/core", "xds_enabled_server"] ] } , "grpc++_unsecure": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_unsecure"] , "srcs": [ "src/cpp/client/insecure_credentials.cc" , "src/cpp/common/insecure_create_auth_context.cc" , "src/cpp/server/insecure_server_credentials.cc" ] , "hdrs": [ ["include/grpc++", "grpc++_public_headers"] , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "channel_arg_names" , "generic_stub_internal" , "global_callback_hook" , "gpr" , "grpc++_base_unsecure" , "grpc++_codegen_proto" , "grpc++_config_proto" , "grpc_core_credentials_header" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_unsecure" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["src/core", "gpr_atm"] , ["src/core", "grpc_insecure_credentials"] ] } , "grpc++_error_details": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_error_details"] , "srcs": ["src/cpp/util/error_details.cc"] , "hdrs": [ "include/grpc++/support/error_details.h" , "include/grpcpp/support/error_details.h" ] , "deps": ["grpc++"] } , "grpc++_alts": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_alts"] , "srcs": ["src/cpp/common/alts_context.cc", "src/cpp/common/alts_util.cc"] , "hdrs": [ "include/grpcpp/security/alts_context.h" , "include/grpcpp/security/alts_util.h" ] , "deps": [ "gpr" , "grpc++" , "grpc_base" , "tsi_alts_credentials" , ["@", "absl", "absl/log", "log"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] , ["third_party/upb", "message"] ] } , "census": { "type": ["@", "rules", "CC", "library"] , "name": ["census"] , "srcs": ["src/core/ext/filters/census/grpc_context.cc"] , "hdrs": [["include/grpc", "census_headers"]] , "deps": [ "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , ["src/core", "arena"] ] } , "gpr_platform": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr_platform"] , "hdrs": [ "include/grpc/impl/codegen/port_platform.h" , "include/grpc/support/port_platform.h" , ["./", "include/grpc", "gpr_public_headers"] ] } , "event_engine_base_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_base_hdrs"] , "hdrs": [ ["include/grpc", "grpc_public_event_engine_headers"] , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "channel_arg_names" , "gpr" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] ] } , "channelz": { "type": ["@", "rules", "CC", "library"] , "name": ["channelz"] , "srcs": [ "src/core/channelz/channel_trace.cc" , "src/core/channelz/channelz.cc" , "src/core/channelz/channelz_registry.cc" ] , "hdrs": [ "src/core/channelz/channel_trace.h" , "src/core/channelz/channelz.h" , "src/core/channelz/channelz_registry.h" ] , "deps": [ "exec_ctx" , "gpr" , "grpc_public_hdrs" , "parse_address" , "ref_counted_ptr" , "sockaddr_utils" , "uri" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "channel_args"] , ["src/core", "connectivity_state"] , ["src/core", "json"] , ["src/core", "json_writer"] , ["src/core", "per_cpu"] , ["src/core", "ref_counted"] , ["src/core", "resolved_address"] , ["src/core", "slice"] , ["src/core", "time"] , ["src/core", "useful"] ] } , "dynamic_annotations": { "type": ["@", "rules", "CC", "library"] , "name": ["dynamic_annotations"] , "hdrs": ["src/core/lib/iomgr/dynamic_annotations.h"] , "deps": ["gpr_public_hdrs"] } , "call_combiner": { "type": ["@", "rules", "CC", "library"] , "name": ["call_combiner"] , "srcs": ["src/core/lib/iomgr/call_combiner.cc"] , "hdrs": ["src/core/lib/iomgr/call_combiner.h"] , "deps": [ "dynamic_annotations" , "exec_ctx" , "gpr" , "ref_counted_ptr" , "stats" , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["src/core", "closure"] , ["src/core", "gpr_atm"] , ["src/core", "ref_counted"] , ["src/core", "stats_data"] ] } , "resource_quota_api": { "type": ["@", "rules", "CC", "library"] , "name": ["resource_quota_api"] , "srcs": ["src/core/lib/resource_quota/api.cc"] , "hdrs": ["src/core/lib/resource_quota/api.h"] , "deps": [ "channel_arg_names" , "config" , "event_engine_base_hdrs" , "exec_ctx" , "gpr_public_hdrs" , "grpc_public_hdrs" , "ref_counted_ptr" , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "memory_quota"] , ["src/core", "resource_quota"] , ["src/core", "thread_quota"] ] } , "byte_buffer": { "type": ["@", "rules", "CC", "library"] , "name": ["byte_buffer"] , "srcs": [ "src/core/lib/surface/byte_buffer.cc" , "src/core/lib/surface/byte_buffer_reader.cc" ] , "deps": [ "exec_ctx" , "gpr_public_hdrs" , "grpc_public_hdrs" , ["@", "absl", "absl/log", "check"] , ["src/core", "compression"] , ["src/core", "slice"] ] } , "iomgr": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr"] , "srcs": [ "src/core/lib/iomgr/cfstream_handle.cc" , "src/core/lib/iomgr/dualstack_socket_posix.cc" , "src/core/lib/iomgr/endpoint.cc" , "src/core/lib/iomgr/endpoint_cfstream.cc" , "src/core/lib/iomgr/endpoint_pair_posix.cc" , "src/core/lib/iomgr/endpoint_pair_windows.cc" , "src/core/lib/iomgr/error_cfstream.cc" , "src/core/lib/iomgr/ev_apple.cc" , "src/core/lib/iomgr/ev_epoll1_linux.cc" , "src/core/lib/iomgr/ev_poll_posix.cc" , "src/core/lib/iomgr/ev_posix.cc" , "src/core/lib/iomgr/fork_posix.cc" , "src/core/lib/iomgr/fork_windows.cc" , "src/core/lib/iomgr/iocp_windows.cc" , "src/core/lib/iomgr/iomgr.cc" , "src/core/lib/iomgr/iomgr_posix.cc" , "src/core/lib/iomgr/iomgr_posix_cfstream.cc" , "src/core/lib/iomgr/iomgr_windows.cc" , "src/core/lib/iomgr/lockfree_event.cc" , "src/core/lib/iomgr/polling_entity.cc" , "src/core/lib/iomgr/pollset.cc" , "src/core/lib/iomgr/pollset_set_windows.cc" , "src/core/lib/iomgr/pollset_windows.cc" , "src/core/lib/iomgr/resolve_address.cc" , "src/core/lib/iomgr/resolve_address_posix.cc" , "src/core/lib/iomgr/resolve_address_windows.cc" , "src/core/lib/iomgr/socket_factory_posix.cc" , "src/core/lib/iomgr/socket_utils_common_posix.cc" , "src/core/lib/iomgr/socket_utils_linux.cc" , "src/core/lib/iomgr/socket_utils_posix.cc" , "src/core/lib/iomgr/socket_windows.cc" , "src/core/lib/iomgr/systemd_utils.cc" , "src/core/lib/iomgr/tcp_client.cc" , "src/core/lib/iomgr/tcp_client_cfstream.cc" , "src/core/lib/iomgr/tcp_client_posix.cc" , "src/core/lib/iomgr/tcp_client_windows.cc" , "src/core/lib/iomgr/tcp_posix.cc" , "src/core/lib/iomgr/tcp_server.cc" , "src/core/lib/iomgr/tcp_server_posix.cc" , "src/core/lib/iomgr/tcp_server_utils_posix_common.cc" , "src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc" , "src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc" , "src/core/lib/iomgr/tcp_server_windows.cc" , "src/core/lib/iomgr/tcp_windows.cc" , "src/core/lib/iomgr/unix_sockets_posix.cc" , "src/core/lib/iomgr/unix_sockets_posix_noop.cc" , "src/core/lib/iomgr/vsock.cc" , "src/core/lib/iomgr/wakeup_fd_eventfd.cc" , "src/core/lib/iomgr/wakeup_fd_nospecial.cc" , "src/core/lib/iomgr/wakeup_fd_pipe.cc" , "src/core/lib/iomgr/wakeup_fd_posix.cc" , "src/core/lib/iomgr/event_engine_shims/closure.cc" , "src/core/lib/iomgr/event_engine_shims/endpoint.cc" , "src/core/lib/iomgr/event_engine_shims/tcp_client.cc" , "src/core/util/gethostname_fallback.cc" , "src/core/util/gethostname_host_name_max.cc" , "src/core/util/gethostname_sysconf.cc" ] , "hdrs": [ "src/core/lib/iomgr/block_annotate.h" , "src/core/lib/iomgr/cfstream_handle.h" , "src/core/lib/iomgr/endpoint.h" , "src/core/lib/iomgr/endpoint_cfstream.h" , "src/core/lib/iomgr/endpoint_pair.h" , "src/core/lib/iomgr/error_cfstream.h" , "src/core/lib/iomgr/ev_apple.h" , "src/core/lib/iomgr/ev_epoll1_linux.h" , "src/core/lib/iomgr/ev_poll_posix.h" , "src/core/lib/iomgr/ev_posix.h" , "src/core/lib/iomgr/iocp_windows.h" , "src/core/lib/iomgr/iomgr.h" , "src/core/lib/iomgr/lockfree_event.h" , "src/core/lib/iomgr/nameser.h" , "src/core/lib/iomgr/polling_entity.h" , "src/core/lib/iomgr/pollset.h" , "src/core/lib/iomgr/pollset_set_windows.h" , "src/core/lib/iomgr/pollset_windows.h" , "src/core/lib/iomgr/python_util.h" , "src/core/lib/iomgr/resolve_address.h" , "src/core/lib/iomgr/resolve_address_impl.h" , "src/core/lib/iomgr/resolve_address_posix.h" , "src/core/lib/iomgr/resolve_address_windows.h" , "src/core/lib/iomgr/sockaddr.h" , "src/core/lib/iomgr/sockaddr_posix.h" , "src/core/lib/iomgr/sockaddr_windows.h" , "src/core/lib/iomgr/socket_factory_posix.h" , "src/core/lib/iomgr/socket_utils_posix.h" , "src/core/lib/iomgr/socket_windows.h" , "src/core/lib/iomgr/systemd_utils.h" , "src/core/lib/iomgr/tcp_client.h" , "src/core/lib/iomgr/tcp_client_posix.h" , "src/core/lib/iomgr/tcp_posix.h" , "src/core/lib/iomgr/tcp_server.h" , "src/core/lib/iomgr/tcp_server_utils_posix.h" , "src/core/lib/iomgr/tcp_windows.h" , "src/core/lib/iomgr/unix_sockets_posix.h" , "src/core/lib/iomgr/vsock.h" , "src/core/lib/iomgr/wakeup_fd_pipe.h" , "src/core/lib/iomgr/wakeup_fd_posix.h" , "src/core/lib/iomgr/event_engine_shims/closure.h" , "src/core/lib/iomgr/event_engine_shims/endpoint.h" , "src/core/lib/iomgr/event_engine_shims/tcp_client.h" , "src/core/util/gethostname.h" , ["include/grpc", "grpc_public_event_engine_headers"] , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "byte_buffer" , "channel_arg_names" , "config_vars" , "debug_location" , "exec_ctx" , "gpr" , "grpc_core_credentials_header" , "grpc_public_hdrs" , "grpc_trace" , "iomgr_buffer_list" , "iomgr_internal_errqueue" , "iomgr_timer" , "orphanable" , "parse_address" , "resource_quota_api" , "sockaddr_utils" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/utility", "utility"] , ["src/core", "1999"] , ["src/core", "channel_args"] , ["src/core", "channel_args_endpoint_config"] , ["src/core", "closure"] , ["src/core", "construct_destruct"] , ["src/core", "context"] , ["src/core", "default_event_engine"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "event_engine_common"] , ["src/core", "event_engine_extensions"] , ["src/core", "event_engine_memory_allocator_factory"] , ["src/core", "event_engine_query_extensions"] , ["src/core", "event_engine_shim"] , ["src/core", "event_engine_tcp_socket_utils"] , ["src/core", "event_log"] , ["src/core", "experiments"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "grpc_sockaddr"] , ["src/core", "init_internally"] , ["src/core", "iomgr_fwd"] , ["src/core", "iomgr_port"] , ["src/core", "memory_quota"] , ["src/core", "no_destruct"] , ["src/core", "pollset_set"] , ["src/core", "posix_event_engine_base_hdrs"] , ["src/core", "posix_event_engine_endpoint"] , ["src/core", "resolved_address"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_cast"] , ["src/core", "slice_refcount"] , ["src/core", "socket_mutator"] , ["src/core", "stats_data"] , ["src/core", "strerror"] , ["src/core", "time"] , ["src/core", "useful"] , ["src/core", "windows_event_engine"] , ["src/core", "windows_event_engine_listener"] ] } , "call_tracer": { "type": ["@", "rules", "CC", "library"] , "name": ["call_tracer"] , "srcs": ["src/core/telemetry/call_tracer.cc"] , "hdrs": ["src/core/telemetry/call_tracer.h"] , "deps": [ "gpr" , "tcp_tracer" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "arena"] , ["src/core", "call_final_info"] , ["src/core", "channel_args"] , ["src/core", "context"] , ["src/core", "error"] , ["src/core", "metadata_batch"] , ["src/core", "ref_counted_string"] , ["src/core", "slice_buffer"] ] } , "channel": { "type": ["@", "rules", "CC", "library"] , "name": ["channel"] , "srcs": ["src/core/lib/surface/channel.cc"] , "hdrs": ["src/core/lib/surface/channel.h"] , "deps": [ "channel_arg_names" , "channelz" , "cpp_impl_of" , "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "grpc_public_hdrs" , "grpc_trace" , "ref_counted_ptr" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "arena"] , ["src/core", "call_arena_allocator"] , ["src/core", "call_destination"] , ["src/core", "channel_args"] , ["src/core", "channel_stack_type"] , ["src/core", "compression"] , ["src/core", "connectivity_state"] , ["src/core", "iomgr_fwd"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "stats_data"] , ["src/core", "time"] ] } , "legacy_channel": { "type": ["@", "rules", "CC", "library"] , "name": ["legacy_channel"] , "srcs": ["src/core/lib/surface/legacy_channel.cc"] , "hdrs": ["src/core/lib/surface/legacy_channel.h"] , "deps": [ "channel" , "channelz" , "config" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_client_channel" , "ref_counted_ptr" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "arena"] , ["src/core", "call_arena_allocator"] , ["src/core", "channel_args"] , ["src/core", "channel_args_endpoint_config"] , ["src/core", "channel_fwd"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "closure"] , ["src/core", "dual_ref_counted"] , ["src/core", "error"] , ["src/core", "init_internally"] , ["src/core", "iomgr_fwd"] , ["src/core", "metrics"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "stats_data"] , ["src/core", "time"] ] } , "channel_create": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_create"] , "srcs": ["src/core/lib/surface/channel_create.cc"] , "hdrs": ["src/core/lib/surface/channel_create.h"] , "deps": [ "channel" , "channel_arg_names" , "channelz" , "config" , "gpr" , "grpc_base" , "grpc_client_channel" , "grpc_public_hdrs" , "legacy_channel" , "ref_counted_ptr" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "arena"] , ["src/core", "channel_args"] , ["src/core", "channel_stack_type"] , ["src/core", "direct_channel"] , ["src/core", "experiments"] , ["src/core", "iomgr_fwd"] , ["src/core", "ref_counted"] , ["src/core", "slice"] , ["src/core", "stats_data"] ] } , "server": { "type": ["@", "rules", "CC", "library"] , "name": ["server"] , "srcs": ["src/core/server/server.cc"] , "hdrs": ["src/core/server/server.h"] , "deps": [ "call_combiner" , "call_tracer" , "channel" , "channel_arg_names" , "channelz" , "config" , "cpp_impl_of" , "debug_location" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_trace" , "iomgr" , "legacy_channel" , "orphanable" , "promise" , "ref_counted_ptr" , "sockaddr_utils" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "activity"] , ["src/core", "arena_promise"] , ["src/core", "cancel_callback"] , ["src/core", "channel_args"] , ["src/core", "channel_args_preconditioning"] , ["src/core", "channel_fwd"] , ["src/core", "channel_stack_type"] , ["src/core", "closure"] , ["src/core", "connection_quota"] , ["src/core", "connectivity_state"] , ["src/core", "context"] , ["src/core", "dual_ref_counted"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "experiments"] , ["src/core", "interception_chain"] , ["src/core", "iomgr_fwd"] , ["src/core", "map"] , ["src/core", "metadata_batch"] , ["src/core", "pipe"] , ["src/core", "poll"] , ["src/core", "pollset_set"] , ["src/core", "random_early_detection"] , ["src/core", "resolved_address"] , ["src/core", "seq"] , ["src/core", "server_interface"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "status_helper"] , ["src/core", "time"] , ["src/core", "try_join"] , ["src/core", "try_seq"] , ["src/core", "useful"] ] } , "grpc_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_base"] , "srcs": [ "src/core/lib/channel/channel_stack.cc" , "src/core/lib/channel/channel_stack_builder_impl.cc" , "src/core/lib/channel/connected_channel.cc" , "src/core/lib/channel/promise_based_filter.cc" , "src/core/lib/channel/status_util.cc" , "src/core/lib/compression/message_compress.cc" , "src/core/lib/surface/call.cc" , "src/core/lib/surface/call_details.cc" , "src/core/lib/surface/call_log_batch.cc" , "src/core/lib/surface/call_utils.cc" , "src/core/lib/surface/client_call.cc" , "src/core/lib/surface/completion_queue.cc" , "src/core/lib/surface/completion_queue_factory.cc" , "src/core/lib/surface/event_string.cc" , "src/core/lib/surface/filter_stack_call.cc" , "src/core/lib/surface/lame_client.cc" , "src/core/lib/surface/metadata_array.cc" , "src/core/lib/surface/server_call.cc" , "src/core/lib/surface/validate_metadata.cc" , "src/core/lib/surface/version.cc" , "src/core/lib/transport/transport.cc" , "src/core/lib/transport/transport_op_string.cc" ] , "hdrs": [ "src/core/lib/channel/channel_stack.h" , "src/core/lib/channel/channel_stack_builder_impl.h" , "src/core/lib/channel/connected_channel.h" , "src/core/lib/channel/promise_based_filter.h" , "src/core/lib/channel/status_util.h" , "src/core/lib/compression/message_compress.h" , "src/core/lib/surface/call.h" , "src/core/lib/surface/call_test_only.h" , "src/core/lib/surface/call_utils.h" , "src/core/lib/surface/client_call.h" , "src/core/lib/surface/completion_queue.h" , "src/core/lib/surface/completion_queue_factory.h" , "src/core/lib/surface/event_string.h" , "src/core/lib/surface/filter_stack_call.h" , "src/core/lib/surface/init.h" , "src/core/lib/surface/lame_client.h" , "src/core/lib/surface/server_call.h" , "src/core/lib/surface/validate_metadata.h" , "src/core/lib/transport/transport.h" , ["include/grpc", "grpc_public_event_engine_headers"] , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "call_combiner" , "call_tracer" , "channel" , "channel_arg_names" , "channel_stack_builder" , "channelz" , "config" , "cpp_impl_of" , "debug_location" , "exec_ctx" , "gpr" , "grpc_core_credentials_header" , "grpc_public_hdrs" , "grpc_trace" , "iomgr" , "iomgr_timer" , "orphanable" , "promise" , "ref_counted_ptr" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/utility", "utility"] , ["@", "zlib", "", "zlib"] , ["src/core", "1999"] , ["src/core", "activity"] , ["src/core", "all_ok"] , ["src/core", "arena"] , ["src/core", "arena_promise"] , ["src/core", "atomic_utils"] , ["src/core", "bitset"] , ["src/core", "blackboard"] , ["src/core", "call_destination"] , ["src/core", "call_filters"] , ["src/core", "call_final_info"] , ["src/core", "call_finalization"] , ["src/core", "call_spine"] , ["src/core", "cancel_callback"] , ["src/core", "channel_args"] , ["src/core", "channel_args_preconditioning"] , ["src/core", "channel_fwd"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "closure"] , ["src/core", "compression"] , ["src/core", "connectivity_state"] , ["src/core", "context"] , ["src/core", "default_event_engine"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "event_engine_common"] , ["src/core", "event_engine_context"] , ["src/core", "experiments"] , ["src/core", "filter_args"] , ["src/core", "for_each"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "gpr_spinlock"] , ["src/core", "if"] , ["src/core", "iomgr_fwd"] , ["src/core", "latch"] , ["src/core", "latent_see"] , ["src/core", "loop"] , ["src/core", "map"] , ["src/core", "match"] , ["src/core", "message"] , ["src/core", "metadata"] , ["src/core", "metadata_batch"] , ["src/core", "metrics"] , ["src/core", "no_destruct"] , ["src/core", "pipe"] , ["src/core", "poll"] , ["src/core", "promise_status"] , ["src/core", "race"] , ["src/core", "ref_counted"] , ["src/core", "seq"] , ["src/core", "server_interface"] , ["src/core", "single_set_ptr"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_cast"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] , ["src/core", "status_flag"] , ["src/core", "status_helper"] , ["src/core", "time"] , ["src/core", "transport_fwd"] , ["src/core", "try_seq"] , ["src/core", "unique_type_name"] , ["src/core", "useful"] ] } , "lb_load_data_store": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_load_data_store"] , "srcs": ["src/cpp/server/load_reporter/load_data_store.cc"] , "hdrs": [ "src/cpp/server/load_reporter/constants.h" , "src/cpp/server/load_reporter/load_data_store.h" ] , "deps": [ "gpr" , "gpr_platform" , "grpc++" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["src/core", "grpc_sockaddr"] ] } , "lb_server_load_reporting_service_server_builder_plugin": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_server_load_reporting_service_server_builder_plugin"] , "srcs": [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_plugin.cc" ] , "hdrs": [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_plugin.h" ] , "deps": ["gpr_platform", "grpc++", "lb_load_reporter_service"] } , "grpcpp_server_load_reporting": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_server_load_reporting"] , "srcs": [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_option.cc" , "src/cpp/server/load_reporter/util.cc" ] , "hdrs": ["include/grpcpp/ext/server_load_reporting.h"] , "deps": [ "channel_arg_names" , "gpr" , "gpr_platform" , "grpc" , "grpc++" , "grpc++_public_hdrs" , "grpc_public_hdrs" , "lb_server_load_reporting_service_server_builder_plugin" , ["@", "absl", "absl/log", "log"] , ["src/core", "lb_server_load_reporting_filter"] ] } , "lb_load_reporter_service": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_load_reporter_service"] , "srcs": ["src/cpp/server/load_reporter/load_reporter_async_service_impl.cc"] , "hdrs": ["src/cpp/server/load_reporter/load_reporter_async_service_impl.h"] , "deps": [ "gpr" , "grpc++" , "lb_load_reporter" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "protobuf", "", "libprotobuf"] , ["src/proto/grpc/lb/v1", "load_reporter_cc_grpc"] ] } , "lb_get_cpu_stats": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_get_cpu_stats"] , "srcs": [ "src/cpp/server/load_reporter/get_cpu_stats_linux.cc" , "src/cpp/server/load_reporter/get_cpu_stats_macos.cc" , "src/cpp/server/load_reporter/get_cpu_stats_unsupported.cc" , "src/cpp/server/load_reporter/get_cpu_stats_windows.cc" ] , "hdrs": ["src/cpp/server/load_reporter/get_cpu_stats.h"] , "deps": ["gpr", "gpr_platform", ["@", "absl", "absl/log", "log"]] } , "lb_load_reporter": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_load_reporter"] , "srcs": ["src/cpp/server/load_reporter/load_reporter.cc"] , "hdrs": [ "src/cpp/server/load_reporter/constants.h" , "src/cpp/server/load_reporter/load_reporter.h" ] , "deps": [ "gpr" , "lb_get_cpu_stats" , "lb_load_data_store" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "opencensus-stats", "", ""] , ["@", "opencensus-tags", "", ""] , ["@", "protobuf", "", "libprotobuf"] , ["src/proto/grpc/lb/v1", "load_reporter_cc_grpc"] ] } , "grpc_security_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_security_base"] , "srcs": [ "src/core/handshaker/security/secure_endpoint.cc" , "src/core/handshaker/security/security_handshaker.cc" , "src/core/lib/security/context/security_context.cc" , "src/core/lib/security/credentials/call_creds_util.cc" , "src/core/lib/security/credentials/composite/composite_credentials.cc" , "src/core/lib/security/credentials/credentials.cc" , "src/core/lib/security/credentials/plugin/plugin_credentials.cc" , "src/core/lib/security/security_connector/security_connector.cc" , "src/core/lib/security/transport/client_auth_filter.cc" , "src/core/lib/security/transport/server_auth_filter.cc" ] , "hdrs": [ "src/core/handshaker/security/secure_endpoint.h" , "src/core/handshaker/security/security_handshaker.h" , "src/core/lib/security/context/security_context.h" , "src/core/lib/security/credentials/call_creds_util.h" , "src/core/lib/security/credentials/composite/composite_credentials.h" , "src/core/lib/security/credentials/credentials.h" , "src/core/lib/security/credentials/plugin/plugin_credentials.h" , "src/core/lib/security/security_connector/security_connector.h" , "src/core/lib/security/transport/auth_filters.h" , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "channel_arg_names" , "channelz" , "config" , "debug_location" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_public_hdrs" , "grpc_trace" , "handshaker" , "iomgr" , "orphanable" , "promise" , "ref_counted_ptr" , "resource_quota_api" , "stats" , "tsi_base" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "activity"] , ["src/core", "arena"] , ["src/core", "arena_promise"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] , ["src/core", "closure"] , ["src/core", "connection_context"] , ["src/core", "context"] , ["src/core", "error"] , ["src/core", "event_engine_memory_allocator"] , ["src/core", "gpr_atm"] , ["src/core", "handshaker_factory"] , ["src/core", "handshaker_registry"] , ["src/core", "iomgr_fwd"] , ["src/core", "memory_quota"] , ["src/core", "metadata_batch"] , ["src/core", "poll"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "seq"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] , ["src/core", "status_helper"] , ["src/core", "try_seq"] , ["src/core", "unique_type_name"] , ["src/core", "useful"] ] } , "tsi_base": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_base"] , "srcs": [ "src/core/tsi/transport_security.cc" , "src/core/tsi/transport_security_grpc.cc" ] , "hdrs": [ "src/core/tsi/transport_security.h" , "src/core/tsi/transport_security_grpc.h" , "src/core/tsi/transport_security_interface.h" ] , "deps": ["gpr", "grpc_public_hdrs", "grpc_trace"] } , "grpc_core_credentials_header": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_core_credentials_header"] , "hdrs": [["include/grpc", "grpc_core_credentials_header"]] } , "alts_util": { "type": ["@", "rules", "CC", "library"] , "name": ["alts_util"] , "srcs": [ "src/core/lib/security/credentials/alts/check_gcp_environment.cc" , "src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc" , "src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc" , "src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc" , "src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc" , "src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc" , "src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc" , "src/core/tsi/alts/handshaker/transport_security_common_api.cc" ] , "hdrs": [ ["include/grpc", "grpc_secure_public_headers"] , "src/core/lib/security/credentials/alts/check_gcp_environment.h" , "src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h" , "src/core/tsi/alts/handshaker/transport_security_common_api.h" ] , "deps": [ "gpr" , "grpc_core_credentials_header" , "grpc_public_hdrs" , ["@", "absl", "absl/log", "log"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "tsi": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi"] , "deps": [ "gpr" , "tsi_alts_frame_protector" , "tsi_base" , "tsi_fake_credentials" , ["@", "absl", "absl/strings", "strings"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["src/core", "tsi_local_credentials"] , ["src/core", "useful"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "grpc++_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_base"] , "srcs": [ "src/cpp/client/insecure_credentials.cc" , "src/cpp/client/secure_credentials.cc" , "src/cpp/common/auth_property_iterator.cc" , "src/cpp/common/secure_auth_context.cc" , "src/cpp/common/secure_create_auth_context.cc" , "src/cpp/common/tls_certificate_provider.cc" , "src/cpp/common/tls_certificate_verifier.cc" , "src/cpp/common/tls_credentials_options.cc" , "src/cpp/server/insecure_server_credentials.cc" , "src/cpp/server/secure_server_credentials.cc" , ["", "grpcxx_sources"] ] , "hdrs": [ "src/cpp/client/secure_credentials.h" , "src/cpp/common/secure_auth_context.h" , "src/cpp/server/secure_server_credentials.h" , ["", "grpcxx_headers"] , ["include/grpc++", "grpc++_public_headers"] , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "channel_arg_names" , "channel_stack_builder" , "config" , "exec_ctx" , "generic_stub_internal" , "global_callback_hook" , "gpr" , "grpc" , "grpc++_codegen_proto" , "grpc++_config_proto" , "grpc_base" , "grpc_core_credentials_header" , "grpc_credentials_util" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_service_config_impl" , "grpc_trace" , "grpcpp_backend_metric_recorder" , "grpcpp_call_metric_recorder" , "grpcpp_status" , "iomgr" , "iomgr_timer" , "ref_counted_ptr" , "resource_quota_api" , "server" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "arena"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "closure"] , ["src/core", "default_event_engine"] , ["src/core", "env"] , ["src/core", "error"] , ["src/core", "experiments"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "grpc_audit_logging"] , ["src/core", "grpc_backend_metric_provider"] , ["src/core", "grpc_crl_provider"] , ["src/core", "grpc_service_config"] , ["src/core", "grpc_tls_credentials"] , ["src/core", "grpc_transport_chttp2_server"] , ["src/core", "grpc_transport_inproc"] , ["src/core", "json"] , ["src/core", "json_reader"] , ["src/core", "load_file"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_refcount"] , ["src/core", "socket_mutator"] , ["src/core", "status_helper"] , ["src/core", "thread_quota"] , ["src/core", "time"] , ["src/core", "useful"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "grpc++_base_unsecure": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_base_unsecure"] , "srcs": [["", "grpcxx_sources"]] , "hdrs": [ ["", "grpcxx_headers"] , ["include/grpc++", "grpc++_public_headers"] , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "channel_arg_names" , "channel_stack_builder" , "config" , "exec_ctx" , "generic_stub_internal" , "global_callback_hook" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_health_upb" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_service_config_impl" , "grpc_trace" , "grpc_transport_chttp2" , "grpc_unsecure" , "grpcpp_backend_metric_recorder" , "grpcpp_call_metric_recorder" , "grpcpp_status" , "iomgr" , "iomgr_timer" , "ref_counted_ptr" , "resource_quota_api" , "server" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "arena"] , ["src/core", "channel_args"] , ["src/core", "channel_init"] , ["src/core", "closure"] , ["src/core", "default_event_engine"] , ["src/core", "error"] , ["src/core", "experiments"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "grpc_backend_metric_provider"] , ["src/core", "grpc_insecure_credentials"] , ["src/core", "grpc_service_config"] , ["src/core", "grpc_transport_chttp2_server"] , ["src/core", "grpc_transport_inproc"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "socket_mutator"] , ["src/core", "thread_quota"] , ["src/core", "time"] , ["src/core", "useful"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "grpc++_codegen_proto": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_codegen_proto"] , "hdrs": [ ["./", "include/grpc++", "grpc++_codegen_proto_headers"] , ["./", "include/grpcpp", "grpcpp_codegen_proto_headers"] ] , "deps": [ "grpc++_config_proto" , "grpc++_public_hdrs" , "grpcpp_status" , ["@", "absl", "absl/strings", "cord"] , ["@", "protobuf", "", "libprotobuf"] ] } , "grpc++_config_proto": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_config_proto"] , "hdrs": [ ["include/grpc++", "grpc++_config_proto_headers"] , ["include/grpcpp", "grpcpp_config_proto_headers"] ] , "deps": [ ["@", "absl", "absl/status", "status"] , ["@", "protobuf", "", "libprotobuf"] ] } , "grpc++_reflection": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_reflection"] , "srcs": [ "src/cpp/ext/proto_server_reflection.cc" , "src/cpp/ext/proto_server_reflection_plugin.cc" ] , "hdrs": [ "src/cpp/ext/proto_server_reflection.h" , "include/grpc++/ext/proto_server_reflection_plugin.h" , "include/grpcpp/ext/proto_server_reflection_plugin.h" ] , "deps": [ "config_vars" , "grpc++" , "grpc++_config_proto" , ["@", "protobuf", "", "libprotobuf"] , ["src/proto/grpc/reflection/v1", "reflection_proto"] , ["src/proto/grpc/reflection/v1alpha", "reflection_proto"] ] } , "grpcpp_call_metric_recorder": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_call_metric_recorder"] , "hdrs": [["include/grpcpp", "grpcpp_call_metric_recorder_headers"]] , "deps": [ "grpc++_public_hdrs" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpcpp_backend_metric_recorder": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_backend_metric_recorder"] , "srcs": ["src/cpp/server/backend_metric_recorder.cc"] , "hdrs": [ "src/cpp/server/backend_metric_recorder.h" , ["include/grpcpp", "grpcpp_backend_metric_recorder_headers"] ] , "deps": [ "gpr" , "grpc++_public_hdrs" , "grpc_trace" , "grpcpp_call_metric_recorder" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "grpc_backend_metric_data"] , ["src/core", "grpc_backend_metric_provider"] ] } , "grpcpp_orca_service": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_orca_service"] , "srcs": ["src/cpp/server/orca/orca_service.cc"] , "hdrs": ["src/cpp/server/orca/orca_service.h"] , "deps": [ "debug_location" , "exec_ctx" , "gpr" , "grpc++" , "grpc_base" , "grpcpp_backend_metric_recorder" , "ref_counted_ptr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "default_event_engine"] , ["src/core", "grpc_backend_metric_data"] , ["src/core", "ref_counted"] , ["src/core", "time"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "grpcpp_channelz": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_channelz"] , "srcs": [ "src/cpp/server/channelz/channelz_service.cc" , "src/cpp/server/channelz/channelz_service_plugin.cc" ] , "hdrs": [ "src/cpp/server/channelz/channelz_service.h" , "include/grpcpp/ext/channelz_service_plugin.h" ] , "deps": [ "gpr" , "grpc" , "grpc++" , "grpc++_config_proto" , ["@", "protobuf", "", "libprotobuf"] , ["src/proto/grpc/channelz", "channelz_proto"] ] } , "grpcpp_csds": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_csds"] , "srcs": ["src/cpp/server/csds/csds.cc"] , "hdrs": ["src/cpp/server/csds/csds.h"] , "deps": [ "gpr" , "grpc" , "grpc++_base" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["src/proto/grpc/testing/xds/v3", "csds_cc_grpc"] ] } , "grpcpp_admin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_admin"] , "srcs": ["src/cpp/server/admin/admin_services.cc"] , "hdrs": ["include/grpcpp/ext/admin_services.h"] , "deps": [ "gpr" , "grpc++" , "grpcpp_channelz" , "grpcpp_csds" , ["@", "absl", "absl/memory", "memory"] ] } , "grpc++_test": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_test"] , "srcs": ["src/cpp/client/channel_test_peer.cc"] , "hdrs": [ "include/grpc++/test/mock_stream.h" , "include/grpc++/test/server_context_test_spouse.h" , "include/grpcpp/test/channel_test_peer.h" , "include/grpcpp/test/client_context_test_peer.h" , "include/grpcpp/test/default_reactor_test_peer.h" , "include/grpcpp/test/mock_stream.h" , "include/grpcpp/test/server_context_test_spouse.h" ] , "deps": ["channel", "grpc++", "grpc_base", ["@", "gtest", "", ""]] } , "grpc_opencensus_plugin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_opencensus_plugin"] , "srcs": [ "src/cpp/ext/filters/census/client_filter.cc" , "src/cpp/ext/filters/census/context.cc" , "src/cpp/ext/filters/census/grpc_plugin.cc" , "src/cpp/ext/filters/census/measures.cc" , "src/cpp/ext/filters/census/rpc_encoding.cc" , "src/cpp/ext/filters/census/server_call_tracer.cc" , "src/cpp/ext/filters/census/views.cc" ] , "hdrs": [ "include/grpcpp/opencensus.h" , "src/cpp/ext/filters/census/client_filter.h" , "src/cpp/ext/filters/census/context.h" , "src/cpp/ext/filters/census/grpc_plugin.h" , "src/cpp/ext/filters/census/measures.h" , "src/cpp/ext/filters/census/open_census_call_tracer.h" , "src/cpp/ext/filters/census/rpc_encoding.h" , "src/cpp/ext/filters/census/server_call_tracer.h" ] , "deps": [ "call_tracer" , "config" , "gpr" , "grpc++_base" , "grpc_base" , "grpc_public_hdrs" , "tcp_tracer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/base", "endian"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "opencensus-stats", "", ""] , ["@", "opencensus-tags", "", ""] , ["@", "opencensus-tags-context_util", "", ""] , ["@", "opencensus-trace", "", ""] , ["@", "opencensus-trace-context_util", "", ""] , ["@", "opencensus-trace-propagation", "", ""] , ["@", "opencensus-trace-span_context", "", ""] , ["src/core", "arena"] , ["src/core", "arena_promise"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] , ["src/core", "channel_stack_type"] , ["src/core", "context"] , ["src/core", "error"] , ["src/core", "experiments"] , ["src/core", "logging_filter"] , ["src/core", "metadata_batch"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_refcount"] ] } , "grpcpp_gcp_observability": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_gcp_observability"] , "hdrs": ["include/grpcpp/ext/gcp_observability.h"] , "deps": [["src/cpp/ext/gcp", "observability"]] } , "grpcpp_csm_observability": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_csm_observability"] , "hdrs": ["include/grpcpp/ext/csm_observability.h"] , "deps": ["grpcpp_otel_plugin", ["src/cpp/ext/csm", "csm_observability"]] } , "grpcpp_otel_plugin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_otel_plugin"] , "hdrs": ["include/grpcpp/ext/otel_plugin.h"] , "deps": ["grpc++", ["src/cpp/ext/otel", "otel_plugin"]] } , "generic_stub_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["generic_stub_internal"] , "hdrs": ["include/grpcpp/impl/generic_stub_internal.h"] , "deps": ["grpc++_public_hdrs"] } , "generic_stub": { "type": ["@", "rules", "CC", "library"] , "name": ["generic_stub"] , "hdrs": ["include/grpcpp/generic/generic_stub.h"] , "deps": ["generic_stub_internal"] } , "generic_stub_callback": { "type": ["@", "rules", "CC", "library"] , "name": ["generic_stub_callback"] , "hdrs": ["include/grpcpp/generic/generic_stub_callback.h"] , "deps": ["generic_stub_internal"] } , "async_generic_service": { "type": ["@", "rules", "CC", "library"] , "name": ["async_generic_service"] , "hdrs": ["include/grpcpp/generic/async_generic_service.h"] , "deps": ["grpc++_public_hdrs"] } , "callback_generic_service": { "type": ["@", "rules", "CC", "library"] , "name": ["callback_generic_service"] , "hdrs": ["include/grpcpp/generic/callback_generic_service.h"] , "deps": ["grpc++_public_hdrs"] } , "work_serializer": { "type": ["@", "rules", "CC", "library"] , "name": ["work_serializer"] , "srcs": ["src/core/util/work_serializer.cc"] , "hdrs": ["src/core/util/work_serializer.h"] , "deps": [ "debug_location" , "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "grpc_trace" , "orphanable" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["src/core", "experiments"] , ["src/core", "latent_see"] , ["src/core", "stats_data"] ] } , "grpc_trace": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_trace"] , "srcs": ["src/core/lib/debug/trace.cc", "src/core/lib/debug/trace_flags.cc"] , "hdrs": [ "src/core/lib/debug/trace.h" , "src/core/lib/debug/trace_flags.h" , "src/core/lib/debug/trace_impl.h" ] , "deps": [ "config_vars" , "gpr" , "grpc_public_hdrs" , ["@", "absl", "absl/strings", "strings"] , ["src/core", "glob"] , ["src/core", "no_destruct"] ] } , "load_config": { "type": ["@", "rules", "CC", "library"] , "name": ["load_config"] , "srcs": ["src/core/config/load_config.cc"] , "hdrs": ["src/core/config/load_config.h"] , "deps": [ "gpr_platform" , ["@", "absl", "absl/flags", "flag"] , ["@", "absl", "absl/flags", "marshalling"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "env"] ] } , "config_vars": { "type": "export" , "target": "config_vars (unexported)" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "PKG_CONFIG_ARGS" ] } , "config_vars (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["config_vars"] , "srcs": [ "src/core/config/config_vars.cc" , "src/core/config/config_vars_non_generated.cc" ] , "hdrs": ["src/core/config/config_vars.h"] , "deps": [ "gpr_platform" , "load_config" , ["@", "absl", "absl/flags", "flag"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "srcs": ["src/core/config/core_configuration.cc"] , "hdrs": ["src/core/config/core_configuration.h"] , "deps": [ "gpr" , "grpc_resolver" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["src/core", "certificate_provider_registry"] , ["src/core", "channel_args_preconditioning"] , ["src/core", "channel_creds_registry"] , ["src/core", "channel_init"] , ["src/core", "handshaker_registry"] , ["src/core", "lb_policy_registry"] , ["src/core", "proxy_mapper_registry"] , ["src/core", "service_config_parser"] ] } , "debug_location": { "type": ["@", "rules", "CC", "library"] , "name": ["debug_location"] , "hdrs": ["src/core/util/debug_location.h"] , "deps": ["gpr_platform", ["@", "absl", "absl/strings", "strings"]] } , "orphanable": { "type": ["@", "rules", "CC", "library"] , "name": ["orphanable"] , "hdrs": ["src/core/util/orphanable.h"] , "deps": [ "debug_location" , "gpr_platform" , "ref_counted_ptr" , ["src/core", "down_cast"] , ["src/core", "ref_counted"] ] } , "promise": { "type": ["@", "rules", "CC", "library"] , "name": ["promise"] , "hdrs": ["src/core/lib/promise/promise.h"] , "deps": [ "gpr_platform" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "poll"] , ["src/core", "promise_like"] ] } , "ref_counted_ptr": { "type": ["@", "rules", "CC", "library"] , "name": ["ref_counted_ptr"] , "hdrs": ["src/core/util/ref_counted_ptr.h"] , "deps": [ "debug_location" , "gpr_platform" , ["@", "absl", "absl/hash", "hash"] , ["src/core", "down_cast"] ] } , "handshaker": { "type": ["@", "rules", "CC", "library"] , "name": ["handshaker"] , "srcs": ["src/core/handshaker/handshaker.cc"] , "hdrs": ["src/core/handshaker/handshaker.h"] , "deps": [ "debug_location" , "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , "iomgr" , "orphanable" , "ref_counted_ptr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["src/core", "channel_args"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "ref_counted"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "status_helper"] , ["src/core", "time"] ] } , "http_connect_handshaker": { "type": ["@", "rules", "CC", "library"] , "name": ["http_connect_handshaker"] , "srcs": ["src/core/handshaker/http_connect/http_connect_handshaker.cc"] , "hdrs": ["src/core/handshaker/http_connect/http_connect_handshaker.h"] , "deps": [ "config" , "debug_location" , "exec_ctx" , "gpr" , "grpc_base" , "handshaker" , "httpcli" , "iomgr" , "ref_counted_ptr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "channel_args"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "handshaker_factory"] , ["src/core", "handshaker_registry"] , ["src/core", "iomgr_fwd"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] ] } , "exec_ctx": { "type": ["@", "rules", "CC", "library"] , "name": ["exec_ctx"] , "srcs": [ "src/core/lib/iomgr/combiner.cc" , "src/core/lib/iomgr/exec_ctx.cc" , "src/core/lib/iomgr/executor.cc" , "src/core/lib/iomgr/iomgr_internal.cc" ] , "hdrs": [ "src/core/lib/iomgr/combiner.h" , "src/core/lib/iomgr/exec_ctx.h" , "src/core/lib/iomgr/executor.h" , "src/core/lib/iomgr/iomgr_internal.h" ] , "deps": [ "debug_location" , "gpr" , "grpc_public_hdrs" , "grpc_trace" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "experiments"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_spinlock"] , ["src/core", "latent_see"] , ["src/core", "time"] , ["src/core", "useful"] ] } , "sockaddr_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["sockaddr_utils"] , "srcs": ["src/core/lib/address_utils/sockaddr_utils.cc"] , "hdrs": ["src/core/lib/address_utils/sockaddr_utils.h"] , "deps": [ "gpr" , "uri" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "grpc_sockaddr"] , ["src/core", "iomgr_port"] , ["src/core", "resolved_address"] ] } , "iomgr_timer": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr_timer"] , "srcs": [ "src/core/lib/iomgr/timer.cc" , "src/core/lib/iomgr/timer_generic.cc" , "src/core/lib/iomgr/timer_heap.cc" , "src/core/lib/iomgr/timer_manager.cc" ] , "hdrs": [ "src/core/lib/iomgr/iomgr.h" , "src/core/lib/iomgr/timer.h" , "src/core/lib/iomgr/timer_generic.h" , "src/core/lib/iomgr/timer_heap.h" , "src/core/lib/iomgr/timer_manager.h" ] , "deps": [ "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "gpr_platform" , "grpc_trace" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "closure"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "gpr_spinlock"] , ["src/core", "iomgr_port"] , ["src/core", "time"] , ["src/core", "time_averaged_stats"] , ["src/core", "useful"] ] } , "iomgr_internal_errqueue": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr_internal_errqueue"] , "srcs": ["src/core/lib/iomgr/internal_errqueue.cc"] , "hdrs": ["src/core/lib/iomgr/internal_errqueue.h"] , "deps": [ "gpr" , ["@", "absl", "absl/log", "log"] , ["src/core", "iomgr_port"] , ["src/core", "strerror"] ] } , "iomgr_buffer_list": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr_buffer_list"] , "srcs": ["src/core/lib/iomgr/buffer_list.cc"] , "hdrs": ["src/core/lib/iomgr/buffer_list.h"] , "deps": [ "gpr" , "iomgr_internal_errqueue" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "error"] , ["src/core", "iomgr_port"] ] } , "uri": { "type": ["@", "rules", "CC", "library"] , "name": ["uri"] , "srcs": ["src/core/util/uri.cc"] , "hdrs": ["src/core/util/uri.h"] , "deps": [ "gpr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] ] } , "parse_address": { "type": ["@", "rules", "CC", "library"] , "name": ["parse_address"] , "srcs": [ "src/core/lib/address_utils/parse_address.cc" , "src/core/util/grpc_if_nametoindex_posix.cc" , "src/core/util/grpc_if_nametoindex_unsupported.cc" ] , "hdrs": [ "src/core/lib/address_utils/parse_address.h" , "src/core/util/grpc_if_nametoindex.h" ] , "deps": [ "gpr" , "uri" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "error"] , ["src/core", "grpc_sockaddr"] , ["src/core", "iomgr_port"] , ["src/core", "resolved_address"] , ["src/core", "status_helper"] ] } , "backoff": { "type": ["@", "rules", "CC", "library"] , "name": ["backoff"] , "srcs": ["src/core/util/backoff.cc"] , "hdrs": ["src/core/util/backoff.h"] , "deps": [ "gpr_platform" , ["@", "absl", "absl/random", "random"] , ["src/core", "experiments"] , ["src/core", "time"] ] } , "stats": { "type": ["@", "rules", "CC", "library"] , "name": ["stats"] , "srcs": ["src/core/telemetry/stats.cc"] , "hdrs": ["src/core/telemetry/stats.h"] , "deps": [ "gpr" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "span"] , ["src/core", "histogram_view"] , ["src/core", "no_destruct"] , ["src/core", "stats_data"] ] } , "channel_stack_builder": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_stack_builder"] , "srcs": ["src/core/lib/channel/channel_stack_builder.cc"] , "hdrs": ["src/core/lib/channel/channel_stack_builder.h"] , "deps": [ "gpr" , "ref_counted_ptr" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] , ["src/core", "channel_stack_type"] ] } , "grpc_service_config_impl": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_service_config_impl"] , "srcs": ["src/core/service_config/service_config_impl.cc"] , "hdrs": ["src/core/service_config/service_config_impl.h"] , "deps": [ "config" , "gpr" , "ref_counted_ptr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "channel_args"] , ["src/core", "grpc_service_config"] , ["src/core", "json"] , ["src/core", "json_args"] , ["src/core", "json_object_loader"] , ["src/core", "json_reader"] , ["src/core", "json_writer"] , ["src/core", "service_config_parser"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "validation_errors"] ] } , "endpoint_addresses": { "type": ["@", "rules", "CC", "library"] , "name": ["endpoint_addresses"] , "srcs": ["src/core/resolver/endpoint_addresses.cc"] , "hdrs": ["src/core/resolver/endpoint_addresses.h"] , "deps": [ "gpr" , "gpr_platform" , "sockaddr_utils" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "resolved_address"] , ["src/core", "useful"] ] } , "server_address": { "type": ["@", "rules", "CC", "library"] , "name": ["server_address"] , "hdrs": ["src/core/resolver/server_address.h"] , "deps": ["endpoint_addresses", "gpr_public_hdrs"] } , "grpc_resolver": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver"] , "srcs": ["src/core/resolver/resolver.cc", "src/core/resolver/resolver_registry.cc"] , "hdrs": [ "src/core/resolver/resolver.h" , "src/core/resolver/resolver_factory.h" , "src/core/resolver/resolver_registry.h" ] , "deps": [ "endpoint_addresses" , "gpr" , "grpc_trace" , "orphanable" , "ref_counted_ptr" , "server_address" , "uri" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "grpc_service_config"] , ["src/core", "iomgr_fwd"] ] } , "oob_backend_metric": { "type": ["@", "rules", "CC", "library"] , "name": ["oob_backend_metric"] , "srcs": ["src/core/load_balancing/oob_backend_metric.cc"] , "hdrs": [ "src/core/load_balancing/oob_backend_metric.h" , "src/core/load_balancing/oob_backend_metric_internal.h" ] , "deps": [ "channelz" , "debug_location" , "exec_ctx" , "gpr" , "grpc_client_channel" , "grpc_public_hdrs" , "grpc_trace" , "orphanable" , "ref_counted_ptr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "backend_metric_parser"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "grpc_backend_metric_data"] , ["src/core", "iomgr_fwd"] , ["src/core", "pollset_set"] , ["src/core", "slice"] , ["src/core", "subchannel_interface"] , ["src/core", "time"] , ["src/core", "unique_type_name"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "lb_child_policy_handler": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_child_policy_handler"] , "srcs": ["src/core/load_balancing/child_policy_handler.cc"] , "hdrs": ["src/core/load_balancing/child_policy_handler.h"] , "deps": [ "config" , "debug_location" , "gpr_public_hdrs" , "grpc_public_hdrs" , "grpc_trace" , "orphanable" , "ref_counted_ptr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "connectivity_state"] , ["src/core", "delegating_helper"] , ["src/core", "lb_policy"] , ["src/core", "lb_policy_registry"] , ["src/core", "pollset_set"] , ["src/core", "resolved_address"] , ["src/core", "subchannel_interface"] ] } , "grpc_client_channel": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_client_channel"] , "srcs": [ "src/core/client_channel/client_channel.cc" , "src/core/client_channel/client_channel_factory.cc" , "src/core/client_channel/client_channel_filter.cc" , "src/core/client_channel/client_channel_plugin.cc" , "src/core/client_channel/dynamic_filters.cc" , "src/core/client_channel/global_subchannel_pool.cc" , "src/core/client_channel/load_balanced_call_destination.cc" , "src/core/client_channel/local_subchannel_pool.cc" , "src/core/client_channel/retry_filter.cc" , "src/core/client_channel/retry_filter_legacy_call_data.cc" , "src/core/client_channel/subchannel.cc" , "src/core/client_channel/subchannel_stream_client.cc" ] , "hdrs": [ "src/core/client_channel/client_channel.h" , "src/core/client_channel/client_channel_factory.h" , "src/core/client_channel/client_channel_filter.h" , "src/core/client_channel/dynamic_filters.h" , "src/core/client_channel/global_subchannel_pool.h" , "src/core/client_channel/load_balanced_call_destination.h" , "src/core/client_channel/local_subchannel_pool.h" , "src/core/client_channel/retry_filter.h" , "src/core/client_channel/retry_filter_legacy_call_data.h" , "src/core/client_channel/subchannel.h" , "src/core/client_channel/subchannel_interface_internal.h" , "src/core/client_channel/subchannel_stream_client.h" ] , "deps": [ "backoff" , "call_combiner" , "call_tracer" , "channel" , "channel_arg_names" , "channelz" , "config" , "debug_location" , "endpoint_addresses" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_resolver" , "grpc_security_base" , "grpc_service_config_impl" , "grpc_trace" , "iomgr" , "lb_child_policy_handler" , "orphanable" , "promise" , "ref_counted_ptr" , "sockaddr_utils" , "stats" , "uri" , "work_serializer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "arena"] , ["src/core", "arena_promise"] , ["src/core", "backend_metric_parser"] , ["src/core", "blackboard"] , ["src/core", "call_destination"] , ["src/core", "call_filters"] , ["src/core", "call_spine"] , ["src/core", "cancel_callback"] , ["src/core", "channel_args"] , ["src/core", "channel_args_endpoint_config"] , ["src/core", "channel_fwd"] , ["src/core", "channel_init"] , ["src/core", "channel_stack_type"] , ["src/core", "client_channel_args"] , ["src/core", "client_channel_backup_poller"] , ["src/core", "client_channel_internal_header"] , ["src/core", "client_channel_service_config"] , ["src/core", "closure"] , ["src/core", "config_selector"] , ["src/core", "connectivity_state"] , ["src/core", "construct_destruct"] , ["src/core", "context"] , ["src/core", "dual_ref_counted"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "exec_ctx_wakeup_scheduler"] , ["src/core", "experiments"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "grpc_backend_metric_data"] , ["src/core", "grpc_channel_idle_filter"] , ["src/core", "grpc_service_config"] , ["src/core", "idle_filter_state"] , ["src/core", "init_internally"] , ["src/core", "interception_chain"] , ["src/core", "iomgr_fwd"] , ["src/core", "json"] , ["src/core", "latch"] , ["src/core", "lb_metadata"] , ["src/core", "lb_policy"] , ["src/core", "lb_policy_registry"] , ["src/core", "loop"] , ["src/core", "map"] , ["src/core", "memory_quota"] , ["src/core", "metadata"] , ["src/core", "metadata_batch"] , ["src/core", "metrics"] , ["src/core", "observable"] , ["src/core", "pipe"] , ["src/core", "poll"] , ["src/core", "pollset_set"] , ["src/core", "proxy_mapper_registry"] , ["src/core", "ref_counted"] , ["src/core", "resolved_address"] , ["src/core", "resource_quota"] , ["src/core", "retry_interceptor"] , ["src/core", "retry_service_config"] , ["src/core", "retry_throttle"] , ["src/core", "seq"] , ["src/core", "single_set_ptr"] , ["src/core", "sleep"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] , ["src/core", "status_helper"] , ["src/core", "subchannel_connector"] , ["src/core", "subchannel_interface"] , ["src/core", "subchannel_pool_interface"] , ["src/core", "time"] , ["src/core", "try_seq"] , ["src/core", "unique_type_name"] , ["src/core", "useful"] , ["src/core/ext/upb-gen", "upb-gen-lib"] ] } , "grpc_resolver_dns_ares": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_dns_ares"] , "srcs": [ "src/core/resolver/dns/c_ares/dns_resolver_ares.cc" , "src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc" , "src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc" , "src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc" , "src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc" , "src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" ] , "hdrs": [ "src/core/resolver/dns/c_ares/dns_resolver_ares.h" , "src/core/resolver/dns/c_ares/grpc_ares_ev_driver.h" , "src/core/resolver/dns/c_ares/grpc_ares_wrapper.h" ] , "deps": [ "backoff" , "channel_arg_names" , "config" , "config_vars" , "debug_location" , "endpoint_addresses" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_grpclb_balancer_addresses" , "grpc_resolver" , "grpc_service_config_impl" , "grpc_trace" , "iomgr" , "iomgr_timer" , "orphanable" , "parse_address" , "ref_counted_ptr" , "sockaddr_utils" , "uri" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "cares", "", "ares"] , ["src/core", "channel_args"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "grpc_service_config"] , ["src/core", "grpc_sockaddr"] , ["src/core", "iomgr_fwd"] , ["src/core", "iomgr_port"] , ["src/core", "polling_resolver"] , ["src/core", "pollset_set"] , ["src/core", "resolved_address"] , ["src/core", "service_config_helper"] , ["src/core", "slice"] , ["src/core", "status_helper"] , ["src/core", "time"] , ["third_party/address_sorting", "address_sorting"] ] } , "httpcli": { "type": ["@", "rules", "CC", "library"] , "name": ["httpcli"] , "srcs": [ "src/core/util/http_client/format_request.cc" , "src/core/util/http_client/httpcli.cc" , "src/core/util/http_client/parser.cc" ] , "hdrs": [ "src/core/util/http_client/format_request.h" , "src/core/util/http_client/httpcli.h" , "src/core/util/http_client/parser.h" ] , "deps": [ "config" , "debug_location" , "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_security_base" , "grpc_trace" , "handshaker" , "iomgr" , "orphanable" , "ref_counted_ptr" , "resource_quota_api" , "uri" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "channel_args"] , ["src/core", "channel_args_preconditioning"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "event_engine_common"] , ["src/core", "event_engine_tcp_socket_utils"] , ["src/core", "handshaker_registry"] , ["src/core", "iomgr_fwd"] , ["src/core", "pollset_set"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "status_helper"] , ["src/core", "tcp_connect_handshaker"] , ["src/core", "time"] ] } , "grpc_alts_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_alts_credentials"] , "srcs": [ "src/core/lib/security/credentials/alts/alts_credentials.cc" , "src/core/lib/security/security_connector/alts/alts_security_connector.cc" ] , "hdrs": [ "src/core/lib/security/credentials/alts/alts_credentials.h" , "src/core/lib/security/security_connector/alts/alts_security_connector.h" ] , "deps": [ "alts_util" , "channel_arg_names" , "debug_location" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_public_hdrs" , "grpc_security_base" , "handshaker" , "iomgr" , "promise" , "ref_counted_ptr" , "tsi_alts_credentials" , "tsi_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "arena_promise"] , ["src/core", "channel_args"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "iomgr_fwd"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "unique_type_name"] , ["src/core", "useful"] ] } , "tsi_fake_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_fake_credentials"] , "srcs": ["src/core/tsi/fake_transport_security.cc"] , "hdrs": ["src/core/tsi/fake_transport_security.h"] , "deps": [ "gpr" , "tsi_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["src/core", "dump_args"] , ["src/core", "slice"] , ["src/core", "useful"] ] } , "grpc_jwt_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_jwt_credentials"] , "srcs": [ "src/core/lib/security/credentials/jwt/json_token.cc" , "src/core/lib/security/credentials/jwt/jwt_credentials.cc" , "src/core/lib/security/credentials/jwt/jwt_verifier.cc" ] , "hdrs": [ "src/core/lib/security/credentials/jwt/json_token.h" , "src/core/lib/security/credentials/jwt/jwt_credentials.h" , "src/core/lib/security/credentials/jwt/jwt_verifier.h" ] , "deps": [ "exec_ctx" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_credentials_util" , "grpc_security_base" , "grpc_trace" , "httpcli" , "iomgr" , "orphanable" , "promise" , "ref_counted_ptr" , "uri" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["src/core", "arena_promise"] , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "httpcli_ssl_credentials"] , ["src/core", "iomgr_fwd"] , ["src/core", "json"] , ["src/core", "json_reader"] , ["src/core", "json_writer"] , ["src/core", "metadata_batch"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "time"] , ["src/core", "tsi_ssl_types"] , ["src/core", "unique_type_name"] , ["src/core", "useful"] ] } , "grpc_credentials_util": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_credentials_util"] , "srcs": [ "src/core/lib/security/credentials/tls/tls_utils.cc" , "src/core/lib/security/security_connector/load_system_roots_fallback.cc" , "src/core/lib/security/security_connector/load_system_roots_supported.cc" , "src/core/lib/security/security_connector/load_system_roots_windows.cc" , "src/core/lib/security/util/json_util.cc" ] , "hdrs": [ "src/core/lib/security/credentials/tls/tls_utils.h" , "src/core/lib/security/security_connector/load_system_roots.h" , "src/core/lib/security/security_connector/load_system_roots_supported.h" , "src/core/lib/security/util/json_util.h" ] , "deps": [ "config_vars" , "gpr" , "grpc_base" , "grpc_security_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "error"] , ["src/core", "json"] , ["src/core", "load_file"] , ["src/core", "useful"] ] } , "tsi_alts_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_alts_credentials"] , "srcs": [ "src/core/tsi/alts/handshaker/alts_handshaker_client.cc" , "src/core/tsi/alts/handshaker/alts_shared_resource.cc" , "src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc" , "src/core/tsi/alts/handshaker/alts_tsi_utils.cc" ] , "hdrs": [ "src/core/tsi/alts/handshaker/alts_handshaker_client.h" , "src/core/tsi/alts/handshaker/alts_shared_resource.h" , "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h" , "src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h" , "src/core/tsi/alts/handshaker/alts_tsi_utils.h" ] , "deps": [ "alts_util" , "channel" , "channel_create" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_security_base" , "tsi_alts_frame_protector" , "tsi_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "closure"] , ["src/core", "env"] , ["src/core", "pollset_set"] , ["src/core", "slice"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "tsi_alts_frame_protector": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_alts_frame_protector"] , "srcs": [ "src/core/tsi/alts/crypt/aes_gcm.cc" , "src/core/tsi/alts/crypt/gsec.cc" , "src/core/tsi/alts/frame_protector/alts_counter.cc" , "src/core/tsi/alts/frame_protector/alts_crypter.cc" , "src/core/tsi/alts/frame_protector/alts_frame_protector.cc" , "src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc" , "src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc" , "src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc" , "src/core/tsi/alts/frame_protector/frame_handler.cc" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc" , "src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc" , "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc" ] , "hdrs": [ "src/core/tsi/alts/crypt/gsec.h" , "src/core/tsi/alts/frame_protector/alts_counter.h" , "src/core/tsi/alts/frame_protector/alts_crypter.h" , "src/core/tsi/alts/frame_protector/alts_frame_protector.h" , "src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h" , "src/core/tsi/alts/frame_protector/frame_handler.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h" , "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h" ] , "deps": [ "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "gpr_platform" , "tsi_base" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/types", "span"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "useful"] ] } , "tsi_ssl_session_cache": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_ssl_session_cache"] , "srcs": [ "src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc" , "src/core/tsi/ssl/session_cache/ssl_session_cache.cc" , "src/core/tsi/ssl/session_cache/ssl_session_openssl.cc" ] , "hdrs": [ "src/core/tsi/ssl/session_cache/ssl_session.h" , "src/core/tsi/ssl/session_cache/ssl_session_cache.h" ] , "deps": [ "cpp_impl_of" , "gpr" , "grpc_public_hdrs" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "ssl", "", "ssl"] , ["src/core", "ref_counted"] , ["src/core", "slice"] ] } , "tsi_ssl_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_ssl_credentials"] , "srcs": [ "src/core/lib/security/security_connector/ssl_utils.cc" , "src/core/tsi/ssl/key_logging/ssl_key_logging.cc" , "src/core/tsi/ssl_transport_security.cc" , "src/core/tsi/ssl_transport_security_utils.cc" ] , "hdrs": [ "src/core/lib/security/security_connector/ssl_utils.h" , "src/core/tsi/ssl/key_logging/ssl_key_logging.h" , "src/core/tsi/ssl_transport_security.h" , "src/core/tsi/ssl_transport_security_utils.h" ] , "deps": [ "channel_arg_names" , "config_vars" , "gpr" , "grpc_base" , "grpc_core_credentials_header" , "grpc_credentials_util" , "grpc_public_hdrs" , "grpc_security_base" , "ref_counted_ptr" , "tsi_base" , "tsi_ssl_session_cache" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["src/core", "channel_args"] , ["src/core", "error"] , ["src/core", "grpc_crl_provider"] , ["src/core", "grpc_transport_chttp2_alpn"] , ["src/core", "load_file"] , ["src/core", "ref_counted"] , ["src/core", "slice"] , ["src/core", "tsi_ssl_types"] , ["src/core", "useful"] ] } , "grpc_http_filters": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_http_filters"] , "srcs": [ "src/core/ext/filters/http/client/http_client_filter.cc" , "src/core/ext/filters/http/http_filters_plugin.cc" , "src/core/ext/filters/http/message_compress/compression_filter.cc" , "src/core/ext/filters/http/server/http_server_filter.cc" ] , "hdrs": [ "src/core/ext/filters/http/client/http_client_filter.h" , "src/core/ext/filters/http/message_compress/compression_filter.h" , "src/core/ext/filters/http/server/http_server_filter.h" ] , "deps": [ "call_tracer" , "channel_arg_names" , "config" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , "promise" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "activity"] , ["src/core", "arena"] , ["src/core", "arena_promise"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] , ["src/core", "channel_stack_type"] , ["src/core", "compression"] , ["src/core", "context"] , ["src/core", "experiments"] , ["src/core", "grpc_message_size_filter"] , ["src/core", "latch"] , ["src/core", "latent_see"] , ["src/core", "map"] , ["src/core", "metadata_batch"] , ["src/core", "percent_encoding"] , ["src/core", "pipe"] , ["src/core", "poll"] , ["src/core", "prioritized_race"] , ["src/core", "race"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "status_conversion"] ] } , "grpc_grpclb_balancer_addresses": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_grpclb_balancer_addresses"] , "srcs": ["src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc"] , "hdrs": ["src/core/load_balancing/grpclb/grpclb_balancer_addresses.h"] , "deps": [ "endpoint_addresses" , "gpr_platform" , "grpc_public_hdrs" , ["src/core", "channel_args"] , ["src/core", "useful"] ] } , "xds_client": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_client"] , "srcs": [ "src/core/xds/xds_client/lrs_client.cc" , "src/core/xds/xds_client/xds_api.cc" , "src/core/xds/xds_client/xds_bootstrap.cc" , "src/core/xds/xds_client/xds_client.cc" ] , "hdrs": [ "src/core/xds/xds_client/lrs_client.h" , "src/core/xds/xds_client/xds_api.h" , "src/core/xds/xds_client/xds_bootstrap.h" , "src/core/xds/xds_client/xds_channel_args.h" , "src/core/xds/xds_client/xds_client.h" , "src/core/xds/xds_client/xds_locality.h" , "src/core/xds/xds_client/xds_metrics.h" , "src/core/xds/xds_client/xds_resource_type.h" , "src/core/xds/xds_client/xds_resource_type_impl.h" , "src/core/xds/xds_client/xds_transport.h" ] , "deps": [ "backoff" , "call_tracer" , "debug_location" , "endpoint_addresses" , "event_engine_base_hdrs" , "exec_ctx" , "gpr" , "grpc_trace" , "orphanable" , "ref_counted_ptr" , "uri" , "work_serializer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "default_event_engine"] , ["src/core", "dual_ref_counted"] , ["src/core", "env"] , ["src/core", "grpc_backend_metric_data"] , ["src/core", "json"] , ["src/core", "per_cpu"] , ["src/core", "ref_counted"] , ["src/core", "ref_counted_string"] , ["src/core", "time"] , ["src/core", "upb_utils"] , ["src/core", "useful"] , ["src/core", "xds_backend_metric_propagation"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["src/core/ext/upbdefs-gen", "upbdefs-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "json"] , ["third_party/upb", "mem"] , ["third_party/upb", "reflection"] , ["third_party/upb", "text"] ] } , "grpc_mock_cel": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_mock_cel"] , "hdrs": [ "src/core/lib/security/authorization/mock_cel/activation.h" , "src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h" , "src/core/lib/security/authorization/mock_cel/cel_expression.h" , "src/core/lib/security/authorization/mock_cel/cel_value.h" , "src/core/lib/security/authorization/mock_cel/evaluator_core.h" , "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h" ] , "deps": [ "gpr_public_hdrs" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "span"] , ["src/core/ext/upb-gen", "upb-gen-lib"] ] } , "grpc_resolver_fake": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_fake"] , "srcs": ["src/core/resolver/fake/fake_resolver.cc"] , "hdrs": ["src/core/resolver/fake/fake_resolver.h"] , "deps": [ "config" , "debug_location" , "gpr" , "grpc_public_hdrs" , "grpc_resolver" , "orphanable" , "ref_counted_ptr" , "server_address" , "uri" , "work_serializer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "channel_args"] , ["src/core", "notification"] , ["src/core", "ref_counted"] , ["src/core", "useful"] ] } , "chttp2_frame": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_frame"] , "srcs": ["src/core/ext/transport/chttp2/transport/frame.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/frame.h"] , "deps": [ "gpr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] ] } , "chttp2_legacy_frame": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_legacy_frame"] , "hdrs": ["src/core/ext/transport/chttp2/transport/legacy_frame.h"] , "deps": ["gpr"] } , "hpack_parser_table": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_parser_table"] , "srcs": ["src/core/ext/transport/chttp2/transport/hpack_parser_table.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/hpack_parser_table.h"] , "deps": [ "gpr" , "gpr_platform" , "grpc_trace" , "hpack_parse_result" , "stats" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "hpack_constants"] , ["src/core", "metadata_batch"] , ["src/core", "no_destruct"] , ["src/core", "parsed_metadata"] , ["src/core", "slice"] , ["src/core", "unique_ptr_with_bitset"] ] } , "hpack_parse_result": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_parse_result"] , "srcs": ["src/core/ext/transport/chttp2/transport/hpack_parse_result.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/hpack_parse_result.h"] , "deps": [ "gpr" , "grpc_base" , "ref_counted_ptr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["src/core", "error"] , ["src/core", "hpack_constants"] , ["src/core", "ref_counted"] , ["src/core", "slice"] , ["src/core", "status_helper"] ] } , "hpack_parser": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_parser"] , "srcs": ["src/core/ext/transport/chttp2/transport/hpack_parser.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/hpack_parser.h"] , "deps": [ "call_tracer" , "chttp2_legacy_frame" , "gpr" , "gpr_platform" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , "hpack_parse_result" , "hpack_parser_table" , "stats" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] , ["src/core", "decode_huff"] , ["src/core", "error"] , ["src/core", "hpack_constants"] , ["src/core", "match"] , ["src/core", "metadata_batch"] , ["src/core", "metadata_info"] , ["src/core", "parsed_metadata"] , ["src/core", "random_early_detection"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] ] } , "hpack_encoder": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_encoder"] , "srcs": ["src/core/ext/transport/chttp2/transport/hpack_encoder.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/hpack_encoder.h"] , "deps": [ "call_tracer" , "chttp2_bin_encoder" , "chttp2_legacy_frame" , "chttp2_varint" , "gpr" , "gpr_platform" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["src/core", "hpack_constants"] , ["src/core", "hpack_encoder_table"] , ["src/core", "metadata_batch"] , ["src/core", "metadata_compression_traits"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "time"] , ["src/core", "timeout_encoding"] ] } , "chttp2_bin_encoder": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_bin_encoder"] , "srcs": ["src/core/ext/transport/chttp2/transport/bin_encoder.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/bin_encoder.h"] , "deps": [ "gpr" , "gpr_platform" , ["@", "absl", "absl/log", "check"] , ["src/core", "huffsyms"] , ["src/core", "slice"] ] } , "chttp2_varint": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_varint"] , "srcs": ["src/core/ext/transport/chttp2/transport/varint.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/varint.h"] , "deps": [ "gpr" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] ] } , "chttp2_context_list_entry": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_context_list_entry"] , "hdrs": ["src/core/ext/transport/chttp2/transport/context_list_entry.h"] , "deps": ["gpr", "tcp_tracer"] } , "tcp_tracer": { "type": ["@", "rules", "CC", "library"] , "name": ["tcp_tracer"] , "hdrs": ["src/core/telemetry/tcp_tracer.h"] , "deps": [ "gpr" , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_http2_client_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_varint"] , "srcs": ["src/core/ext/transport/chttp2/transport/http2_client_transport.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/http2_client_transport.h"] , "deps": [ "grpc_base" , "hpack_encoder" , "hpack_parser" , ["src/core", "grpc_promise_endoint"] ] } , "grpc_http2_server_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_varint"] , "srcs": ["src/core/ext/transport/chttp2/transport/http2_server_transport.cc"] , "hdrs": ["src/core/ext/transport/chttp2/transport/http2_server_transport.h"] , "deps": [ "grpc_base" , "hpack_encoder" , "hpack_parser" , ["src/core", "grpc_promise_endoint"] ] } , "grpc_transport_chttp2": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_chttp2"] , "srcs": [ "src/core/ext/transport/chttp2/transport/bin_decoder.cc" , "src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc" , "src/core/ext/transport/chttp2/transport/chttp2_transport.cc" , "src/core/ext/transport/chttp2/transport/frame_data.cc" , "src/core/ext/transport/chttp2/transport/frame_goaway.cc" , "src/core/ext/transport/chttp2/transport/frame_ping.cc" , "src/core/ext/transport/chttp2/transport/frame_rst_stream.cc" , "src/core/ext/transport/chttp2/transport/frame_security.cc" , "src/core/ext/transport/chttp2/transport/frame_settings.cc" , "src/core/ext/transport/chttp2/transport/frame_window_update.cc" , "src/core/ext/transport/chttp2/transport/parsing.cc" , "src/core/ext/transport/chttp2/transport/stream_lists.cc" , "src/core/ext/transport/chttp2/transport/writing.cc" ] , "hdrs": [ "src/core/ext/transport/chttp2/transport/bin_decoder.h" , "src/core/ext/transport/chttp2/transport/call_tracer_wrapper.h" , "src/core/ext/transport/chttp2/transport/chttp2_transport.h" , "src/core/ext/transport/chttp2/transport/frame_data.h" , "src/core/ext/transport/chttp2/transport/frame_goaway.h" , "src/core/ext/transport/chttp2/transport/frame_ping.h" , "src/core/ext/transport/chttp2/transport/frame_rst_stream.h" , "src/core/ext/transport/chttp2/transport/frame_security.h" , "src/core/ext/transport/chttp2/transport/frame_settings.h" , "src/core/ext/transport/chttp2/transport/frame_window_update.h" , "src/core/ext/transport/chttp2/transport/internal.h" , "src/core/ext/transport/chttp2/transport/stream_lists.h" ] , "deps": [ "call_tracer" , "channel_arg_names" , "channelz" , "chttp2_context_list_entry" , "chttp2_legacy_frame" , "chttp2_varint" , "config_vars" , "debug_location" , "exec_ctx" , "gpr" , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" , "hpack_encoder" , "hpack_parser" , "hpack_parser_table" , "httpcli" , "iomgr" , "iomgr_buffer_list" , "ref_counted_ptr" , "stats" , "tcp_tracer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "distributions"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["src/core", "arena"] , ["src/core", "bdp_estimator"] , ["src/core", "bitset"] , ["src/core", "channel_args"] , ["src/core", "chttp2_flow_control"] , ["src/core", "closure"] , ["src/core", "connectivity_state"] , ["src/core", "error"] , ["src/core", "error_utils"] , ["src/core", "event_engine_extensions"] , ["src/core", "event_engine_query_extensions"] , ["src/core", "experiments"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "http2_errors"] , ["src/core", "http2_settings"] , ["src/core", "init_internally"] , ["src/core", "iomgr_fwd"] , ["src/core", "iomgr_port"] , ["src/core", "match"] , ["src/core", "memory_quota"] , ["src/core", "metadata_batch"] , ["src/core", "metadata_info"] , ["src/core", "ping_abuse_policy"] , ["src/core", "ping_callbacks"] , ["src/core", "ping_rate_policy"] , ["src/core", "poll"] , ["src/core", "random_early_detection"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] , ["src/core", "status_conversion"] , ["src/core", "status_helper"] , ["src/core", "time"] , ["src/core", "transport_framing_endpoint_extension"] , ["src/core", "useful"] , ["src/core", "write_size_policy"] ] } , "grpcpp_status": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_status"] , "srcs": ["src/cpp/util/status.cc"] , "hdrs": [ "include/grpc++/impl/codegen/status.h" , "include/grpc++/support/status.h" , "include/grpcpp/impl/codegen/status.h" , "include/grpcpp/impl/status.h" , "include/grpcpp/support/status.h" ] , "deps": ["gpr_platform", "grpc++_public_hdrs", "grpc_public_hdrs"] } , "grpcpp_chaotic_good": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_chaotic_good"] , "srcs": ["src/cpp/ext/chaotic_good.cc"] , "hdrs": ["src/cpp/ext/chaotic_good.h"] , "deps": [ "gpr" , "grpc++_base" , "grpc_public_hdrs" , ["src/core", "chaotic_good_connector"] , ["src/core", "chaotic_good_server"] ] } , "subprocess": { "type": ["@", "rules", "CC", "library"] , "name": ["subprocess"] , "srcs": [ "src/core/util/subprocess_poxis.cc" , "src/core/util/subprocess_windows.cc" ] , "hdrs": ["src/core/util/subprocess.h"] , "deps": [ "gpr" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "span"] , ["src/core", "strerror"] , ["src/core", "tchar"] ] } , "global_callback_hook": { "type": ["@", "rules", "CC", "library"] , "name": ["global_callback_hook"] , "srcs": ["src/cpp/client/global_callback_hook.cc"] , "hdrs": [["include/grpcpp", "global_callback_hook_headers"]] , "deps": [ ["@", "absl", "absl/base", "no_destructor"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] ] } , "grpc_cpp_plugin": { "type": "export" , "target": ["src/compiler", "grpc_cpp_plugin"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "PKG_CONFIG_ARGS" ] } , "grpc_cpp_plugin (no debug)": { "type": "configure" , "target": "grpc_cpp_plugin" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "toolchain": { "type": ["@", "rules", "CC", "install-with-deps"] , "targets": ["grpc_cpp_plugin (no debug)"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.gsl000066400000000000000000000004461516554100600235300ustar00rootroot00000000000000{ "gsl": { "type": "export" , "target": "gsl (unexported)" , "flexible_config": ["OS", "ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG", "ENV"] } , "gsl (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["gsl"] , "hdrs": [["TREE", null, "."]] , "stage": ["gsl"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.json000066400000000000000000000010661516554100600237130ustar00rootroot00000000000000{ "json": { "type": "export" , "target": "json library" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "json library": { "type": ["@", "rules", "CC", "library"] , "name": ["nlohmann_json"] , "hdrs": ["json headers"] } , "json headers": { "type": ["@", "rules", "data", "staged"] , "stage": ["nlohmann"] , "srcs": [["TREE", null, "."]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.lzma000066400000000000000000000264251516554100600237130ustar00rootroot00000000000000{ "lzma": { "type": "export" , "target": "lzma_config" , "doc": ["The LZMA linkable library"] , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" , "ENABLE_SMALL" , "ADDITIONAL_CHECK_TYPES" , "ENABLE_THREADS" , "ENCODERS" , "DECODERS" , "LZIP_DECODER" ] , "config_doc": { "ENABLE_SMALL": ["Boolean. Default value: false. Reduce code size at expense of speed."] , "ADDITIONAL_CHECK_TYPES": ["Boolean. Default value: true. Support additional crc64/sha256 checks."] , "ENABLE_THREADS": ["Boolean. Default value: true. Support threading."] , "ENCODERS": ["Boolean. Default value: true. Include encoder support."] , "DECODERS": ["Boolean. Default value: true. Include decoder support."] , "LZIP_DECODER": ["Boolean. Default value: true. Support lzip decoder."] } } , "lzma_config": { "type": "configure" , "arguments_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "ENABLE_SMALL" , "ENABLE_THREADS" , "ENCODERS" , "DECODERS" , "LZIP_DECODER" ] , "target": "lzma_internal" , "config": { "type": "let*" , "bindings": [ [ "OS" , { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": { "type": "var" , "name": "ARCH" , "default": {"type": "fail", "msg": "Required variable 'ARCH' is not set."} } } ] , [ "ENABLE_SMALL" , {"type": "var", "name": "ENABLE_SMALL", "default": false} ] , [ "ADDITIONAL_CHECK_TYPES" , {"type": "var", "name": "ADDITIONAL_CHECK_TYPES", "default": true} ] , [ "ENABLE_THREADS" , {"type": "var", "name": "ENABLE_THREADS", "default": true} ] , ["ENCODERS", {"type": "var", "name": "ENCODERS", "default": true}] , ["DECODERS", {"type": "var", "name": "DECODERS", "default": true}] , [ "LZIP_DECODER" , {"type": "var", "name": "LZIP_DECODER", "default": true} ] ] , "body": { "type": "env" , "vars": [ "OS" , "TARGET_ARCH" , "ENABLE_SMALL" , "ADDITIONAL_CHECK_TYPES" , "ENABLE_THREADS" , "ENCODERS" , "DECODERS" , "LZIP_DECODER" ] } } } , "lzma_internal": { "type": ["@", "rules", "CC", "library"] , "arguments_config": [ "OS" , "TARGET_ARCH" , "ENABLE_SMALL" , "ADDITIONAL_CHECK_TYPES" , "ENABLE_THREADS" , "ENCODERS" , "DECODERS" , "LZIP_DECODER" ] , "name": ["lzma"] , "pure C": ["YES"] , "cflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["-pthread"] } , "private-cflags": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "ENABLE_SMALL"} , "then": ["-DHAVE_SMALL"] } , { "type": "if" , "cond": {"type": "var", "name": "ADDITIONAL_CHECK_TYPES"} , "then": ["-DHAVE_CHECK_CRC64", "-DHAVE_CHECK_SHA256"] } , { "type": "if" , "cond": {"type": "var", "name": "MATCH_FINDERS", "default": true} , "then": [ "-DHAVE_MF_HC3" , "-DHAVE_MF_HC4" , "-DHAVE_MF_BT2" , "-DHAVE_MF_BT3" , "-DHAVE_MF_BT4" ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["-DMYTHREAD_POSIX"] } , { "type": "if" , "cond": {"type": "var", "name": "ENCODERS"} , "then": [ "-DHAVE_ENCODERS" , "-DHAVE_ENCODER_LZMA1" , "-DHAVE_ENCODER_LZMA2" , "-DHAVE_ENCODER_X86" , "-DHAVE_ENCODER_ARM" , "-DHAVE_ENCODER_ARMTHUMB" , "-DHAVE_ENCODER_ARM64" , "-DHAVE_ENCODER_POWERPC" , "-DHAVE_ENCODER_IA64" , "-DHAVE_ENCODER_SPARC" , "-DHAVE_ENCODER_DELTA" ] } , { "type": "if" , "cond": {"type": "var", "name": "DECODERS"} , "then": [ "-DHAVE_DECODERS" , "-DHAVE_DECODER_LZMA1" , "-DHAVE_DECODER_LZMA2" , "-DHAVE_DECODER_X86" , "-DHAVE_DECODER_ARM" , "-DHAVE_DECODER_ARMTHUMB" , "-DHAVE_DECODER_ARM64" , "-DHAVE_DECODER_POWERPC" , "-DHAVE_DECODER_IA64" , "-DHAVE_DECODER_SPARC" , "-DHAVE_DECODER_DELTA" ] } , { "type": "if" , "cond": {"type": "var", "name": "LZIP_DECODER"} , "then": ["-DHAVE_LZIP_DECODER"] } , [ "-DHAVE_CLOCK_GETTIME" , "-DHAVE_DECL_CLOCK_MONOTONIC" , "-DHAVE_PTHREAD_CONDATTR_SETCLOCK" , "-DTUKLIB_SYMBOL_PREFIX=lzma_" , "-DHAVE_FUNC_ATTRIBUTE_CONSTRUCTOR" , "-DHAVE_CPUID_H" , "-DHAVE_VISIBILITY=0" ] , { "type": "case" , "expr": {"type": "var", "name": "TARGET_ARCH"} , "case": { "x86": ["-DHAVE_IMMINTRIN_H", "-DHAVE__MM_MOVEMASK_EPI8"] , "x86_64": [ "-DHAVE_IMMINTRIN_H" , "-DHAVE__MM_MOVEMASK_EPI8" , "-DHAVE_USABLE_CLMUL" ] } } , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": {"windows": ["-DLZMA_API_STATIC"]} } ] } , "hdrs": [["src/liblzma/api", "public_headers"]] , "srcs": { "type": "++" , "$1": [ [ "src/liblzma/check/check.c" , "src/liblzma/common/block_util.c" , "src/liblzma/common/common.c" , "src/liblzma/common/easy_preset.c" , "src/liblzma/common/filter_common.c" , "src/liblzma/common/hardware_physmem.c" , "src/liblzma/common/index.c" , "src/liblzma/common/stream_flags_common.c" , "src/liblzma/common/string_conversion.c" , "src/liblzma/common/vli_size.c" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_SMALL"} , "then": ["src/liblzma/check/crc32_small.c"] , "else": ["src/liblzma/check/crc32_fast.c", "src/liblzma/check/crc32_table.c"] } , { "type": "if" , "cond": {"type": "var", "name": "ADDITIONAL_CHECK_TYPES"} , "then": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_SMALL"} , "then": ["src/liblzma/check/crc64_small.c"] , "else": [ "src/liblzma/check/crc64_fast.c" , "src/liblzma/check/crc64_table.c" ] } } , { "type": "if" , "cond": {"type": "var", "name": "ADDITIONAL_CHECK_TYPES"} , "then": ["src/liblzma/check/sha256.c"] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": [ "src/common/tuklib_cpucores.c" , "src/liblzma/common/hardware_cputhreads.c" , "src/liblzma/common/outqueue.c" ] } , { "type": "if" , "cond": {"type": "var", "name": "ENCODERS"} , "then": { "type": "++" , "$1": [ [ "src/liblzma/common/alone_encoder.c" , "src/liblzma/common/block_buffer_encoder.c" , "src/liblzma/common/block_encoder.c" , "src/liblzma/common/block_header_encoder.c" , "src/liblzma/common/easy_buffer_encoder.c" , "src/liblzma/common/easy_encoder.c" , "src/liblzma/common/easy_encoder_memusage.c" , "src/liblzma/common/filter_buffer_encoder.c" , "src/liblzma/common/filter_encoder.c" , "src/liblzma/common/filter_flags_encoder.c" , "src/liblzma/common/index_encoder.c" , "src/liblzma/common/stream_buffer_encoder.c" , "src/liblzma/common/stream_encoder.c" , "src/liblzma/common/stream_flags_encoder.c" , "src/liblzma/common/vli_encoder.c" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["src/liblzma/common/stream_encoder_mt.c"] } , [ "src/liblzma/simple/simple_encoder.c" , "src/liblzma/lzma/lzma_encoder.c" , "src/liblzma/lzma/lzma_encoder_optimum_fast.c" , "src/liblzma/lzma/lzma_encoder_optimum_normal.c" , "src/liblzma/lz/lz_encoder.c" , "src/liblzma/lz/lz_encoder_mf.c" , "src/liblzma/rangecoder/price_table.c" , "src/liblzma/lzma/lzma2_encoder.c" , "src/liblzma/delta/delta_encoder.c" , "extra_sources" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_SMALL"} , "then": [] , "else": ["src/liblzma/lzma/fastpos_table.c"] } ] } } , { "type": "if" , "cond": {"type": "var", "name": "DECODERS"} , "then": { "type": "++" , "$1": [ [ "src/liblzma/common/alone_decoder.c" , "src/liblzma/common/auto_decoder.c" , "src/liblzma/common/block_buffer_decoder.c" , "src/liblzma/common/block_decoder.c" , "src/liblzma/common/block_header_decoder.c" , "src/liblzma/common/easy_decoder_memusage.c" , "src/liblzma/common/file_info.c" , "src/liblzma/common/filter_buffer_decoder.c" , "src/liblzma/common/filter_decoder.c" , "src/liblzma/common/filter_flags_decoder.c" , "src/liblzma/common/index_decoder.c" , "src/liblzma/common/index_hash.c" , "src/liblzma/common/stream_buffer_decoder.c" , "src/liblzma/common/stream_decoder.c" , "src/liblzma/common/stream_flags_decoder.c" , "src/liblzma/common/vli_decoder.c" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["src/liblzma/common/stream_decoder_mt.c"] } , [ "src/liblzma/simple/simple_decoder.c" , "src/liblzma/lzma/lzma_decoder.c" , "src/liblzma/lz/lz_decoder.c" , "src/liblzma/lzma/lzma2_decoder.c" , "src/liblzma/delta/delta_decoder.c" , "extra_sources" ] ] } } , { "type": "if" , "cond": {"type": "var", "name": "LZIP_DECODER"} , "then": ["src/liblzma/common/lzip_decoder.c"] } ] } , "private-deps": [ ["src/common", "headers"] , ["src/liblzma/check", "headers"] , ["src/liblzma/common", "headers"] ] } , "extra_sources": { "type": "install" , "deps": [ "src/liblzma/delta/delta_common.c" , "src/liblzma/lzma/lzma_encoder_presets.c" , "src/liblzma/simple/arm.c" , "src/liblzma/simple/arm64.c" , "src/liblzma/simple/armthumb.c" , "src/liblzma/simple/ia64.c" , "src/liblzma/simple/powerpc.c" , "src/liblzma/simple/simple_coder.c" , "src/liblzma/simple/sparc.c" , "src/liblzma/simple/x86.c" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.protobuf000066400000000000000000000040401516554100600245750ustar00rootroot00000000000000{ "protoc": { "type": "export" , "doc": [ "The protobuffer compiler." , "" , "This target typically is used as an implicit dependency of" , "the protobuffer rules." ] , "target": ["src/google/protobuf", "protoc"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "libprotoc": { "type": "export" , "doc": [] , "target": ["src/google/protobuf", "libprotoc"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "libprotobuf": { "type": "export" , "target": ["src/google/protobuf", "libprotobuf"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "libprotobuf_lite": { "type": "export" , "target": ["src/google/protobuf", "libprotobuf_lite"] , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "installed protoc": {"type": ["@", "rules", "CC", "install-with-deps"], "targets": ["protoc"]} , "installed protoc (no debug)": { "type": "configure" , "target": "installed protoc" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "toolchain_headers": {"type": "install", "deps": ["libprotobuf"]} , "toolchain": { "type": "install" , "dirs": [["installed protoc (no debug)", "."], ["toolchain_headers", "include"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.re2000066400000000000000000000030201516554100600234220ustar00rootroot00000000000000{ "re2": { "type": "export" , "target": "re2_internal" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "re2_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["re2"] , "srcs": [ "re2/bitstate.cc" , "re2/compile.cc" , "re2/dfa.cc" , "re2/filtered_re2.cc" , "re2/mimics_pcre.cc" , "re2/nfa.cc" , "re2/onepass.cc" , "re2/parse.cc" , "re2/perl_groups.cc" , "re2/prefilter.cc" , "re2/prefilter_tree.cc" , "re2/prog.cc" , "re2/re2.cc" , "re2/regexp.cc" , "re2/set.cc" , "re2/simplify.cc" , "re2/stringpiece.cc" , "re2/tostring.cc" , "re2/unicode_casefold.cc" , "re2/unicode_groups.cc" , "util/rune.cc" , "util/strutil.cc" ] , "private-hdrs": [ "re2/bitmap256.h" , "re2/pod_array.h" , "re2/prefilter.h" , "re2/prefilter_tree.h" , "re2/prog.h" , "re2/regexp.h" , "re2/sparse_array.h" , "re2/sparse_set.h" , "re2/unicode_casefold.h" , "re2/unicode_groups.h" , "re2/walker-inl.h" , "util/logging.h" , "util/mix.h" , "util/mutex.h" , "util/strutil.h" , "util/utf.h" , "util/util.h" ] , "hdrs": ["re2/filtered_re2.h", "re2/re2.h", "re2/set.h", "re2/stringpiece.h"] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/TARGETS.zlib000066400000000000000000000025001516554100600236740ustar00rootroot00000000000000{ "zlib": { "type": "configure" , "target": "exported zlib" , "arguments_config": ["ARCH", "TARGET_ARCH"] , "config": { "type": "let*" , "bindings": [ [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["TARGET_ARCH"]} } } , "exported zlib": { "type": "export" , "target": "zlibinternal" , "flexible_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "AR" , "ENV" ] } , "zlibinternal": { "type": ["@", "rules", "CC", "library"] , "name": ["z"] , "pkg-name": ["zlib"] , "pure C": ["YES"] , "srcs": [ "adler32.c" , "compress.c" , "crc32.c" , "deflate.c" , "gzclose.c" , "gzlib.c" , "gzread.c" , "gzwrite.c" , "inflate.c" , "infback.c" , "inftrees.c" , "inffast.c" , "trees.c" , "uncompr.c" , "zutil.c" ] , "hdrs": ["zlib.h", "zconf.h"] , "private-hdrs": [ "crc32.h" , "deflate.h" , "gzguts.h" , "inffast.h" , "inffixed.h" , "inflate.h" , "inftrees.h" , "trees.h" , "zutil.h" ] , "private-cflags": ["-Wno-implicit-function-declaration"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/000077500000000000000000000000001516554100600226255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/algorithm/000077500000000000000000000000001516554100600246135ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/algorithm/TARGETS.absl000066400000000000000000000022541516554100600265720ustar00rootroot00000000000000{ "algorithm": { "type": "export" , "target": "algorithm (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "algorithm (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["algorithm"] , "stage": ["absl", "algorithm"] , "hdrs": ["algorithm.h"] , "deps": [["absl/base", "config"]] } , "container": { "type": "export" , "target": "container (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "container (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["container"] , "stage": ["absl", "algorithm"] , "hdrs": ["container.h"] , "deps": [ "algorithm" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/meta", "type_traits"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/base/000077500000000000000000000000001516554100600235375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/base/TARGETS.absl000066400000000000000000000336501516554100600255220ustar00rootroot00000000000000{ "base": { "type": "export" , "target": "base (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "base (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "TOOLCHAIN_CONFIG", "ARCH", "TARGET_ARCH"] , "name": ["base"] , "stage": ["absl", "base"] , "hdrs": [ "call_once.h" , "casts.h" , "internal/cycleclock.h" , "internal/cycleclock_config.h" , "internal/low_level_scheduling.h" , "internal/per_thread_tls.h" , "internal/spinlock.h" , "internal/sysinfo.h" , "internal/thread_identity.h" , "internal/tsan_mutex_interface.h" , "internal/unscaledcycleclock.h" , "internal/unscaledcycleclock_config.h" ] , "srcs": [ "internal/cycleclock.cc" , "internal/spinlock.cc" , "internal/sysinfo.cc" , "internal/thread_identity.cc" , "internal/unscaledcycleclock.cc" ] , "private-ldflags": { "type": "++" , "$1": [ ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] , { "type": "case*" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "case": [["mingw", ["-ladvapi32"]]] } , { "type": "case" , "expr": {"type": "var", "name": "OS", "default": "linux"} , "case": {"linux": ["-lrt"]} } , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } , "deps": [ "atomic_hook" , "base_internal" , "config" , "core_headers" , "cycleclock_internal" , "dynamic_annotations" , "log_severity" , "nullability" , "raw_logging_internal" , "spinlock_wait" , ["absl/meta", "type_traits"] ] } , "atomic_hook": { "type": "export" , "target": "atomic_hook (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "atomic_hook (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["atomic_hook"] , "stage": ["absl", "base"] , "hdrs": ["internal/atomic_hook.h"] , "private-ldflags": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["config", "core_headers"] } , "base_internal": { "type": "export" , "target": "base_internal (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "base_internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["base_internal"] , "stage": ["absl", "base"] , "hdrs": [ "internal/hide_ptr.h" , "internal/identity.h" , "internal/inline_variable.h" , "internal/invoke.h" , "internal/scheduling_mode.h" ] , "deps": ["config", ["absl/meta", "type_traits"]] } , "config": { "type": "export" , "target": "config (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "config (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "stage": ["absl", "base"] , "hdrs": ["config.h", "options.h", "policy_checks.h"] } , "options.h": { "type": ["@", "rules", "patch", "file"] , "src": [["FILE", null, "options.h"]] , "patch": [["@", "patches", "", "options.h.diff"]] } , "core_headers": { "type": "export" , "target": "core_headers (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "core_headers (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["core_headers"] , "stage": ["absl", "base"] , "hdrs": [ "attributes.h" , "const_init.h" , "macros.h" , "optimization.h" , "port.h" , "thread_annotations.h" ] , "deps": ["config"] } , "cycleclock_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["cycleclock_internal"] , "stage": ["absl", "base"] , "hdrs": ["internal/cycleclock_config.h", "internal/unscaledcycleclock_config.h"] , "deps": ["base_internal", "config"] } , "dynamic_annotations": { "type": "export" , "target": "dynamic_annotations (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "dynamic_annotations (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["dynamic_annotations"] , "stage": ["absl", "base"] , "hdrs": ["dynamic_annotations.h", "internal/dynamic_annotations.h"] , "deps": ["config", "core_headers"] } , "log_severity": { "type": "export" , "target": "log_severity (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_severity (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_severity"] , "stage": ["absl", "base"] , "hdrs": ["log_severity.h"] , "srcs": ["log_severity.cc"] , "deps": ["config", "core_headers"] } , "no_destructor": { "type": "export" , "target": "no_destructor (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "no_destructor (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["no_destructor"] , "stage": ["absl", "base"] , "hdrs": ["no_destructor.h"] , "deps": ["config", "nullability"] } , "nullability": { "type": "export" , "target": "nullability (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "nullability (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["nullability"] , "stage": ["absl", "base"] , "hdrs": ["nullability.h", "internal/nullability_impl.h"] , "deps": ["config", "core_headers", ["absl/meta", "type_traits"]] } , "raw_logging_internal": { "type": "export" , "target": "raw_logging_internal (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "raw_logging_internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["raw_logging_internal"] , "stage": ["absl", "base"] , "hdrs": ["internal/raw_logging.h"] , "srcs": ["internal/raw_logging.cc"] , "private-ldflags": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["atomic_hook", "config", "core_headers", "errno_saver", "log_severity"] } , "spinlock_wait": { "type": ["@", "rules", "CC", "library"] , "name": ["spinlock_wait"] , "stage": ["absl", "base"] , "hdrs": ["internal/spinlock_wait.h"] , "srcs": ["internal/spinlock_wait.cc"] , "private-hdrs": [ "internal/spinlock_akaros.inc" , "internal/spinlock_linux.inc" , "internal/spinlock_posix.inc" , "internal/spinlock_win32.inc" ] , "deps": ["base_internal", "core_headers", "errno_saver"] } , "errno_saver": { "type": "export" , "target": "errno_saver (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "errno_saver (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["errno_saver"] , "stage": ["absl", "base"] , "hdrs": ["internal/errno_saver.h"] , "deps": ["config"] } , "endian": { "type": "export" , "target": "endian (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "endian (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["endian"] , "stage": ["absl", "base"] , "hdrs": ["internal/endian.h", "internal/unaligned_access.h"] , "deps": ["base", "config", "core_headers", "nullability"] } , "fast_type_id": { "type": "export" , "target": "fast_type_id (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "fast_type_id (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["fast_type_id"] , "stage": ["absl", "base"] , "hdrs": ["internal/fast_type_id.h"] , "deps": ["config"] } , "strerror": { "type": "export" , "target": "strerror (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "strerror (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["strerror"] , "stage": ["absl", "base"] , "hdrs": ["internal/strerror.h"] , "srcs": ["internal/strerror.cc"] , "deps": ["config", "core_headers", "errno_saver"] } , "throw_delegate": { "type": "export" , "target": "throw_delegate (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "throw_delegate (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["throw_delegate"] , "stage": ["absl", "base"] , "hdrs": ["internal/throw_delegate.h"] , "srcs": ["internal/throw_delegate.cc"] , "deps": ["config", "raw_logging_internal"] } , "malloc_internal": { "type": "export" , "target": "malloc_internal (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "PATCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "malloc_internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["malloc_internal"] , "stage": ["absl", "base"] , "hdrs": ["internal/direct_mmap.h", "internal/low_level_alloc.h"] , "srcs": ["internal/low_level_alloc.cc"] , "private-ldflags": { "type": "++" , "$1": [ ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } , "deps": [ "base" , "base_internal" , "config" , "core_headers" , "dynamic_annotations" , "raw_logging_internal" ] } , "prefetch": { "type": "export" , "target": "prefetch (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "prefetch (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["prefetch"] , "stage": ["absl", "base"] , "hdrs": ["prefetch.h"] , "deps": ["config", "core_headers"] } , "poison": { "type": "export" , "target": "poison (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "poison (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["poison"] , "stage": ["absl", "base"] , "srcs": ["internal/poison.cc"] , "hdrs": ["internal/poison.h"] , "deps": ["config", "core_headers", "malloc_internal"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/cleanup/000077500000000000000000000000001516554100600242545ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/cleanup/TARGETS.absl000066400000000000000000000015671516554100600262410ustar00rootroot00000000000000{ "cleanup": { "type": "export" , "target": "cleanup (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "cleanup (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["cleanup"] , "stage": ["absl", "cleanup"] , "hdrs": ["cleanup.h"] , "deps": [ "cleanup_internal" , ["absl/base", "config"] , ["absl/base", "core_headers"] ] } , "cleanup_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["cleanup_internal"] , "stage": ["absl", "cleanup"] , "hdrs": ["internal/cleanup.h"] , "deps": [ ["absl/base", "base_internal"] , ["absl/base", "core_headers"] , ["absl/utility", "utility"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/container/000077500000000000000000000000001516554100600246075ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/container/TARGETS.absl000066400000000000000000000263321516554100600265710ustar00rootroot00000000000000{ "inlined_vector": { "type": "export" , "target": "inlined_vector (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "inlined_vector (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["inlined_vector"] , "stage": ["absl", "container"] , "hdrs": ["inlined_vector.h"] , "deps": [ "inlined_vector_internal" , ["absl/algorithm", "algorithm"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "throw_delegate"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] ] } , "inlined_vector_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["inlined_vector_internal"] , "stage": ["absl", "container"] , "hdrs": ["internal/inlined_vector.h"] , "deps": [ "compressed_tuple" , ["absl/base", "base_internal"] , ["absl/base", "core_headers"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/types", "span"] ] } , "compressed_tuple": { "type": "export" , "target": "compressed_tuple (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "compressed_tuple (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["compressed_tuple"] , "stage": ["absl", "container"] , "hdrs": ["internal/compressed_tuple.h"] , "deps": [["absl/utility", "utility"]] } , "fixed_array": { "type": "export" , "target": "fixed_array (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "fixed_array (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["fixed_array"] , "stage": ["absl", "container"] , "hdrs": ["fixed_array.h"] , "deps": [ "compressed_tuple" , ["absl/algorithm", "algorithm"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "throw_delegate"] , ["absl/memory", "memory"] ] } , "container_memory": { "type": "export" , "target": "container_memory (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "container_memory (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["container_memory"] , "stage": ["absl", "container"] , "hdrs": ["internal/container_memory.h"] , "deps": [ ["absl/base", "config"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/utility", "utility"] ] } , "layout": { "type": "export" , "target": "layout (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "layout (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["layout"] , "stage": ["absl", "container"] , "hdrs": ["internal/layout.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/debugging", "demangle_internal"] , ["absl/meta", "type_traits"] , ["absl/strings", "strings"] , ["absl/types", "span"] , ["absl/utility", "utility"] ] } , "flat_hash_map": { "type": "export" , "target": "flat_hash_map (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "flat_hash_map (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["flat_hash_map"] , "stage": ["absl", "container"] , "hdrs": ["flat_hash_map.h"] , "deps": [ "container_memory" , "hash_container_defaults" , "raw_hash_map" , ["absl/algorithm", "container"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] ] } , "hash_function_defaults": { "type": "export" , "target": "hash_function_defaults (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "hash_function_defaults (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_function_defaults"] , "stage": ["absl", "container"] , "hdrs": ["internal/hash_function_defaults.h"] , "deps": [ ["absl/base", "config"] , ["absl/hash", "hash"] , ["absl/strings", "cord"] , ["absl/strings", "strings"] ] } , "hash_container_defaults": { "type": "export" , "target": "hash_container_defaults (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "hash_container_defaults (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_container_defaults"] , "stage": ["absl", "container"] , "hdrs": ["hash_container_defaults.h"] , "deps": ["hash_function_defaults", ["absl/base", "config"]] } , "raw_hash_map": { "type": "export" , "target": "raw_hash_map (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "raw_hash_map (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["raw_hash_map"] , "stage": ["absl", "container"] , "hdrs": ["internal/raw_hash_map.h"] , "deps": [ "container_memory" , "raw_hash_set" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "throw_delegate"] ] } , "common": { "type": ["@", "rules", "CC", "library"] , "name": ["common"] , "stage": ["absl", "container"] , "hdrs": ["internal/common.h"] , "deps": [["absl/meta", "type_traits"], ["absl/types", "optional"]] } , "raw_hash_set": { "type": "export" , "target": "raw_hash_set (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "raw_hash_set (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["raw_hash_set"] , "stage": ["absl", "container"] , "hdrs": ["internal/raw_hash_set.h"] , "srcs": ["internal/raw_hash_set.cc"] , "deps": [ "common" , "compressed_tuple" , "container_memory" , "hash_policy_traits" , "hashtable_debug_hooks" , "hashtablez_sampler" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "endian"] , ["absl/base", "prefetch"] , ["absl/base", "raw_logging_internal"] , ["absl/hash", "hash"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/utility", "utility"] ] } , "container_common": { "type": ["@", "rules", "CC", "library"] , "name": ["container_common"] , "stage": ["absl", "container"] , "hdrs": ["internal/common.h"] , "deps": [["absl/meta", "type_traits"]] } , "hash_policy_traits": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_policy_traits"] , "stage": ["absl", "container"] , "hdrs": ["internal/hash_policy_traits.h"] , "deps": ["common_policy_traits", ["absl/meta", "type_traits"]] } , "common_policy_traits": { "type": ["@", "rules", "CC", "library"] , "name": ["common_policy_traits"] , "stage": ["absl", "container"] , "hdrs": ["internal/common_policy_traits.h"] , "deps": [["absl/meta", "type_traits"]] } , "hashtable_debug_hooks": { "type": ["@", "rules", "CC", "library"] , "name": ["hashtable_debug_hooks"] , "stage": ["absl", "container"] , "hdrs": ["internal/hashtable_debug_hooks.h"] , "deps": [["absl/base", "config"]] } , "hashtablez_sampler": { "type": ["@", "rules", "CC", "library"] , "name": ["hashtablez_sampler"] , "stage": ["absl", "container"] , "hdrs": ["internal/hashtablez_sampler.h"] , "srcs": [ "internal/hashtablez_sampler.cc" , "internal/hashtablez_sampler_force_weak_definition.cc" ] , "deps": [ ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "no_destructor"] , ["absl/base", "raw_logging_internal"] , ["absl/debugging", "stacktrace"] , ["absl/memory", "memory"] , ["absl/profiling", "exponential_biased"] , ["absl/profiling", "sample_recorder"] , ["absl/synchronization", "synchronization"] , ["absl/time", "time"] , ["absl/utility", "utility"] ] } , "flat_hash_set": { "type": "export" , "target": "flat_hash_set (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "flat_hash_set (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["flat_hash_set"] , "stage": ["absl", "container"] , "hdrs": ["flat_hash_set.h"] , "deps": [ "container_memory" , "hash_container_defaults" , "raw_hash_set" , ["absl/algorithm", "container"] , ["absl/base", "core_headers"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] ] } , "btree": { "type": "export" , "target": "btree (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "btree (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["btree"] , "stage": ["absl", "container"] , "hdrs": [ "btree_map.h" , "btree_set.h" , "internal/btree.h" , "internal/btree_container.h" ] , "deps": [ "common" , "common_policy_traits" , "compressed_tuple" , "container_memory" , "layout" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] , ["absl/base", "throw_delegate"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/strings", "cord"] , ["absl/strings", "strings"] , ["absl/types", "compare"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/crc/000077500000000000000000000000001516554100600233745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/crc/TARGETS.absl000066400000000000000000000060561516554100600253570ustar00rootroot00000000000000{ "crc_cord_state": { "type": "export" , "target": "crc_cord_state (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "crc_cord_state (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["crc_cord_state"] , "stage": ["absl", "crc"] , "hdrs": ["internal/crc_cord_state.h"] , "srcs": ["internal/crc_cord_state.cc"] , "deps": [ "crc32c" , ["absl/base", "config"] , ["absl/base", "no_destructor"] , ["absl/numeric", "bits"] ] } , "crc32c": { "type": "export" , "target": "crc32c (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "crc32c (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["crc32c"] , "stage": ["absl", "crc"] , "hdrs": [ "crc32c.h" , "internal/crc32c.h" , "internal/crc_memcpy.h" , "internal/crc32c_inline.h" ] , "srcs": [ "crc32c.cc" , "internal/crc_memcpy_fallback.cc" , "internal/crc_memcpy_x86_arm_combined.cc" , "internal/crc_non_temporal_memcpy.cc" ] , "deps": [ "cpu_detect" , "crc_internal" , "non_temporal_memcpy" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "prefetch"] , ["absl/strings", "str_format"] , ["absl/strings", "strings"] ] } , "cpu_detect": { "type": ["@", "rules", "CC", "library"] , "name": ["crc_cpu_detect"] , "stage": ["absl", "crc"] , "hdrs": ["internal/cpu_detect.h"] , "srcs": ["internal/cpu_detect.cc"] , "deps": [["absl/base", "base"], ["absl/base", "config"]] } , "crc_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["crc_internal"] , "stage": ["absl", "crc"] , "hdrs": ["internal/crc.h", "internal/crc32_x86_arm_combined_simd.h"] , "srcs": ["internal/crc.cc", "internal/crc_x86_arm_combined.cc"] , "private-hdrs": ["internal/crc_internal.h"] , "deps": [ "cpu_detect" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "prefetch"] , ["absl/base", "raw_logging_internal"] , ["absl/memory", "memory"] , ["absl/numeric", "bits"] ] } , "non_temporal_memcpy": { "type": ["@", "rules", "CC", "library"] , "name": ["non_temporal_memcpy"] , "stage": ["absl", "crc"] , "hdrs": ["internal/non_temporal_memcpy.h"] , "deps": [ "non_temporal_arm_intrinsics" , ["absl/base", "config"] , ["absl/base", "core_headers"] ] } , "non_temporal_arm_intrinsics": { "type": ["@", "rules", "CC", "library"] , "name": ["non_temporal_arm_intrinsics"] , "stage": ["absl", "crc"] , "hdrs": ["internal/non_temporal_arm_intrinsics.h"] , "deps": [["absl/base", "config"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/debugging/000077500000000000000000000000001516554100600245605ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/debugging/TARGETS.absl000066400000000000000000000171231516554100600265400ustar00rootroot00000000000000{ "symbolize": { "type": "export" , "target": "symbolize (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "symbolize (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["TOOLCHAIN_CONFIG", "ARCH", "TARGET_ARCH"] , "name": ["symbolize"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/symbolize.h", "symbolize.h"] , "srcs": ["symbolize.cc"] , "private-hdrs": [ "symbolize_darwin.inc" , "symbolize_elf.inc" , "symbolize_emscripten.inc" , "symbolize_unimplemented.inc" , "symbolize_win32.inc" ] , "private-ldflags": { "type": "++" , "$1": [ { "type": "case*" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "case": [["mingw", ["-ladvapi32"]]] } , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } , "deps": [ "debugging_internal" , "demangle_internal" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "malloc_internal"] , ["absl/base", "raw_logging_internal"] , ["absl/strings", "strings"] ] } , "examine_stack": { "type": "export" , "target": "examine_stack (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "examine_stack (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_message"] , "stage": ["absl", "debugging"] , "srcs": ["internal/examine_stack.cc"] , "hdrs": ["internal/examine_stack.h"] , "deps": [ "stacktrace" , "symbolize" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] ] } , "stacktrace": { "type": "export" , "target": "stacktrace (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "stacktrace (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["stacktrace"] , "stage": ["absl", "debugging"] , "hdrs": ["stacktrace.h", "internal/stacktrace_x86-inl.inc"] , "srcs": ["stacktrace.cc"] , "private-hdrs": [ "internal/stacktrace_config.h" , "internal/stacktrace_aarch64-inl.inc" , "internal/stacktrace_arm-inl.inc" , "internal/stacktrace_emscripten-inl.inc" , "internal/stacktrace_generic-inl.inc" , "internal/stacktrace_powerpc-inl.inc" , "internal/stacktrace_riscv-inl.inc" , "internal/stacktrace_unimplemented-inl.inc" , "internal/stacktrace_win32-inl.inc" ] , "deps": [ "debugging_internal" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "raw_logging_internal"] ] } , "debugging_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["debugging_internal"] , "stage": ["absl", "debugging"] , "hdrs": [ "internal/address_is_readable.h" , "internal/elf_mem_image.h" , "internal/vdso_support.h" ] , "srcs": [ "internal/address_is_readable.cc" , "internal/elf_mem_image.cc" , "internal/vdso_support.cc" ] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "errno_saver"] , ["absl/base", "raw_logging_internal"] ] } , "demangle_internal": { "type": "export" , "target": "demangle_internal (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "demangle_internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["demangle_internal"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/demangle.h"] , "srcs": ["internal/demangle.cc"] , "deps": [ "demangle_rust" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/numeric", "bits"] ] } , "bounded_utf8_length_sequence": { "type": "export" , "target": "bounded_utf8_length_sequence (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "bounded_utf8_length_sequence (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["bounded_utf8_length_sequence"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/bounded_utf8_length_sequence.h"] , "deps": [["absl/base", "config"], ["absl/numeric", "bits"]] } , "decode_rust_punycode": { "type": "export" , "target": "decode_rust_punycode (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "decode_rust_punycode (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["decode_rust_punycode"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/decode_rust_punycode.h"] , "srcs": ["internal/decode_rust_punycode.cc"] , "deps": [ "bounded_utf8_length_sequence" , "utf8_for_code_point" , ["absl/base", "config"] , ["absl/base", "nullability"] ] } , "demangle_rust": { "type": "export" , "target": "demangle_rust (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "demangle_rust (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["demangle_rust"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/demangle_rust.h"] , "srcs": ["internal/demangle_rust.cc"] , "deps": [ "decode_rust_punycode" , ["absl/base", "config"] , ["absl/base", "core_headers"] ] } , "utf8_for_code_point": { "type": "export" , "target": "utf8_for_code_point (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "utf8_for_code_point (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["utf8_for_code_point"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/utf8_for_code_point.h"] , "srcs": ["internal/utf8_for_code_point.cc"] , "deps": [["absl/base", "config"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/flags/000077500000000000000000000000001516554100600237215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/flags/TARGETS.absl000066400000000000000000000112571516554100600257030ustar00rootroot00000000000000{ "path_util": { "type": ["@", "rules", "CC", "library"] , "name": ["path_util"] , "stage": ["absl", "flags"] , "hdrs": ["internal/path_util.h"] , "deps": [["absl/base", "config"], ["absl/strings", "strings"]] } , "program_name": { "type": ["@", "rules", "CC", "library"] , "name": ["program_name"] , "stage": ["absl", "flags"] , "srcs": ["internal/program_name.cc"] , "hdrs": ["internal/program_name.h"] , "deps": [ "path_util" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] ] } , "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "stage": ["absl", "flags"] , "srcs": ["usage_config.cc"] , "hdrs": ["config.h", "usage_config.h"] , "deps": [ "path_util" , "program_name" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] ] } , "marshalling": { "type": "export" , "target": "marshalling (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "marshalling (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["marshalling"] , "stage": ["absl", "flags"] , "srcs": ["marshalling.cc"] , "hdrs": ["marshalling.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/numeric", "int128"] , ["absl/strings", "str_format"] , ["absl/strings", "strings"] , ["absl/types", "optional"] ] } , "commandlineflag_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["commandlineflag_internal"] , "stage": ["absl", "flags"] , "srcs": ["internal/commandlineflag.cc"] , "hdrs": ["internal/commandlineflag.h"] , "deps": [["absl/base", "config"], ["absl/base", "fast_type_id"]] } , "commandlineflag": { "type": ["@", "rules", "CC", "library"] , "name": ["commandlineflag"] , "stage": ["absl", "flags"] , "srcs": ["commandlineflag.cc"] , "hdrs": ["commandlineflag.h"] , "deps": [ "commandlineflag_internal" , ["absl/base", "config"] , ["absl/base", "fast_type_id"] , ["absl/strings", "strings"] , ["absl/types", "optional"] ] } , "private_handle_accessor": { "type": ["@", "rules", "CC", "library"] , "name": ["private_handle_accessor"] , "stage": ["absl", "flags"] , "srcs": ["internal/private_handle_accessor.cc"] , "hdrs": ["internal/private_handle_accessor.h"] , "deps": [ "commandlineflag" , "commandlineflag_internal" , ["absl/base", "config"] , ["absl/strings", "strings"] ] } , "reflection": { "type": ["@", "rules", "CC", "library"] , "name": ["reflection"] , "stage": ["absl", "flags"] , "srcs": ["reflection.cc"] , "hdrs": ["internal/registry.h", "reflection.h"] , "deps": [ "commandlineflag" , "commandlineflag_internal" , "config" , "private_handle_accessor" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "no_destructor"] , ["absl/container", "flat_hash_map"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] ] } , "flag_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["flag_internal"] , "stage": ["absl", "flags"] , "srcs": ["internal/flag.cc"] , "hdrs": ["internal/flag.h", "internal/sequence_lock.h"] , "deps": [ "commandlineflag" , "commandlineflag_internal" , "config" , "marshalling" , "reflection" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] , ["absl/utility", "utility"] ] } , "flag": { "type": "export" , "target": "flag (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "flag (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["flag"] , "stage": ["absl", "flags"] , "hdrs": ["declare.h", "flag.h"] , "deps": [ "commandlineflag" , "config" , "flag_internal" , "reflection" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/functional/000077500000000000000000000000001516554100600247675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/functional/TARGETS.absl000066400000000000000000000040661516554100600267510ustar00rootroot00000000000000{ "function_ref": { "type": "export" , "target": "function_ref (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "function_ref (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["function_ref"] , "stage": ["absl", "functional"] , "hdrs": ["function_ref.h", "internal/function_ref.h"] , "deps": [ "any_invocable" , ["absl/base", "base_internal"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] ] } , "any_invocable": { "type": "export" , "target": "any_invocable (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "any_invocable (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["any_invocable"] , "stage": ["absl", "functional"] , "hdrs": ["any_invocable.h", "internal/any_invocable.h"] , "deps": [ ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] , ["absl/utility", "utility"] ] } , "bind_front": { "type": "export" , "target": "bind_front_internal" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "bind_front_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["bind_front"] , "stage": ["absl", "functional"] , "hdrs": ["bind_front.h", "internal/front_binder.h"] , "deps": [ ["absl/base", "base_internal"] , ["absl/container", "compressed_tuple"] , ["absl/meta", "type_traits"] , ["absl/utility", "utility"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/hash/000077500000000000000000000000001516554100600235505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/hash/TARGETS.absl000066400000000000000000000031311516554100600255220ustar00rootroot00000000000000{ "hash": { "type": "export" , "target": "hash (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "hash (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["hash"] , "stage": ["absl", "hash"] , "hdrs": ["hash.h", "internal/hash.h"] , "srcs": ["internal/hash.cc"] , "deps": [ "city" , "low_level_hash" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/container", "fixed_array"] , ["absl/functional", "function_ref"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] , ["absl/strings", "strings"] , ["absl/types", "optional"] , ["absl/types", "variant"] , ["absl/utility", "utility"] ] } , "city": { "type": ["@", "rules", "CC", "library"] , "name": ["city"] , "stage": ["absl", "hash"] , "hdrs": ["internal/city.h"] , "srcs": ["internal/city.cc"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] ] } , "low_level_hash": { "type": ["@", "rules", "CC", "library"] , "name": ["low_level_hash"] , "stage": ["absl", "hash"] , "hdrs": ["internal/low_level_hash.h"] , "srcs": ["internal/low_level_hash.cc"] , "deps": [ ["absl/base", "config"] , ["absl/base", "endian"] , ["absl/base", "prefetch"] , ["absl/numeric", "int128"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/log/000077500000000000000000000000001516554100600234065ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/log/TARGETS.absl000066400000000000000000000160531516554100600253670ustar00rootroot00000000000000{ "absl_check": { "type": "export" , "target": "absl_check (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "absl_check (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["absl_check"] , "stage": ["absl", "log"] , "hdrs": ["absl_check.h"] , "deps": [["absl/log/internal", "check_impl"]] } , "absl_log": { "type": "export" , "target": "absl_log (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "absl_log (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["absl_log"] , "stage": ["absl", "log"] , "hdrs": ["absl_log.h"] , "deps": [["absl/log/internal", "log_impl"]] } , "check": { "type": "export" , "target": "check (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "check (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["check"] , "stage": ["absl", "log"] , "hdrs": ["check.h"] , "deps": [ ["absl/log/internal", "check_impl"] , ["absl/log/internal", "check_op"] , ["absl/log/internal", "conditions"] , ["absl/log/internal", "log_message"] , ["absl/log/internal", "strip"] ] } , "die_if_null": { "type": "export" , "target": "die_if_null (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "die_if_null (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["die_if_null"] , "stage": ["absl", "log"] , "srcs": ["die_if_null.cc"] , "hdrs": ["die_if_null.h"] , "deps": [ "log" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] ] } , "globals": { "type": "export" , "target": "globals (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "globals (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["globals"] , "stage": ["absl", "log"] , "srcs": ["globals.cc"] , "hdrs": ["globals.h"] , "deps": [ ["absl/base", "atomic_hook"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/base", "raw_logging_internal"] , ["absl/hash", "hash"] , ["absl/log/internal", "vlog_config"] , ["absl/strings", "strings"] ] } , "initialize": { "type": "export" , "target": "initialize (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "initialize (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["initialize"] , "stage": ["absl", "log"] , "srcs": ["initialize.cc"] , "hdrs": ["initialize.h"] , "deps": [ "globals" , ["absl/base", "config"] , ["absl/log/internal", "globals"] , ["absl/time", "time"] ] } , "log": { "type": "export" , "target": "log (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log"] , "stage": ["absl", "log"] , "hdrs": ["log.h"] , "deps": ["vlog_is_on", ["absl/log/internal", "log_impl"]] } , "log_entry": { "type": "export" , "target": "log_entry (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_entry (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_entry"] , "stage": ["absl", "log"] , "srcs": ["log_entry.cc"] , "hdrs": ["log_entry.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/log/internal", "config"] , ["absl/strings", "strings"] , ["absl/time", "time"] , ["absl/types", "span"] ] } , "log_sink": { "type": "export" , "target": "log_sink (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_sink (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_sink"] , "stage": ["absl", "log"] , "srcs": ["log_sink.cc"] , "hdrs": ["log_sink.h"] , "deps": ["log_entry", ["absl/base", "config"]] } , "log_sink_registry": { "type": "export" , "target": "log_sink_registry (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_sink_registry (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_sink_registry"] , "stage": ["absl", "log"] , "hdrs": ["log_sink_registry.h"] , "deps": [ "log_sink" , ["absl/base", "config"] , ["absl/log/internal", "log_sink_set"] ] } , "absl_vlog_is_on": { "type": "export" , "target": "absl_vlog_is_on (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "absl_vlog_is_on (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["absl_vlog_is_on"] , "stage": ["absl", "log"] , "hdrs": ["absl_vlog_is_on.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/log/internal", "vlog_config"] , ["absl/strings", "strings"] ] } , "vlog_is_on": { "type": ["@", "rules", "CC", "library"] , "name": ["vlog_is_on"] , "stage": ["absl", "log"] , "hdrs": ["vlog_is_on.h"] , "deps": ["absl_vlog_is_on"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/log/internal/000077500000000000000000000000001516554100600252225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/log/internal/TARGETS.absl000066400000000000000000000247731516554100600272130ustar00rootroot00000000000000{ "check_impl": { "type": "export" , "target": "check_impl (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "check_impl (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["check_impl"] , "stage": ["absl", "log", "internal"] , "hdrs": ["check_impl.h"] , "deps": [ "check_op" , "conditions" , "log_message" , "strip" , ["absl/base", "core_headers"] ] } , "check_op": { "type": "export" , "target": "check_op (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "check_op (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["check_op"] , "stage": ["absl", "log", "internal"] , "srcs": ["check_op.cc"] , "hdrs": ["check_op.h"] , "deps": [ "nullguard" , "nullstream" , "strip" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] ] } , "conditions": { "type": "export" , "target": "conditions (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "conditions (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["conditions"] , "stage": ["absl", "log", "internal"] , "srcs": ["conditions.cc"] , "hdrs": ["conditions.h"] , "deps": [ "voidify" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] ] } , "config": { "type": "export" , "target": "config (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "config (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "stage": ["absl", "log", "internal"] , "hdrs": ["config.h"] , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] } , "format": { "type": ["@", "rules", "CC", "library"] , "name": ["format"] , "stage": ["absl", "log", "internal"] , "srcs": ["log_format.cc"] , "hdrs": ["log_format.h"] , "deps": [ "append_truncated" , "config" , "globals" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/strings", "str_format"] , ["absl/strings", "strings"] , ["absl/time", "time"] , ["absl/types", "span"] ] } , "globals": { "type": "export" , "target": "globals (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "globals (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["globals"] , "stage": ["absl", "log", "internal"] , "srcs": ["globals.cc"] , "hdrs": ["globals.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/base", "raw_logging_internal"] , ["absl/strings", "strings"] , ["absl/time", "time"] ] } , "log_impl": { "type": "export" , "target": "log_impl (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_impl (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_impl"] , "stage": ["absl", "log", "internal"] , "hdrs": ["log_impl.h"] , "deps": ["conditions", "log_message", "strip", ["absl/log", "absl_vlog_is_on"]] } , "log_message": { "type": "export" , "target": "log_messages (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "log_messages (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["log_message"] , "stage": ["absl", "log", "internal"] , "srcs": ["log_message.cc"] , "hdrs": ["log_message.h"] , "deps": [ "append_truncated" , "format" , "globals" , "log_sink_set" , "nullguard" , "proto" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "errno_saver"] , ["absl/base", "log_severity"] , ["absl/base", "raw_logging_internal"] , ["absl/base", "strerror"] , ["absl/container", "inlined_vector"] , ["absl/debugging", "examine_stack"] , ["absl/log", "globals"] , ["absl/log", "log_entry"] , ["absl/log", "log_sink"] , ["absl/log", "log_sink_registry"] , ["absl/memory", "memory"] , ["absl/strings", "strings"] , ["absl/time", "time"] , ["absl/types", "span"] ] } , "append_truncated": { "type": ["@", "rules", "CC", "library"] , "name": ["append_truncated"] , "stage": ["absl", "log", "internal"] , "hdrs": ["append_truncated.h"] , "deps": [ ["absl/base", "config"] , ["absl/strings", "strings"] , ["absl/types", "span"] ] } , "log_sink_set": { "type": ["@", "rules", "CC", "library"] , "name": ["log_sink_set"] , "stage": ["absl", "log", "internal"] , "srcs": ["log_sink_set.cc"] , "hdrs": ["log_sink_set.h"] , "deps": [ "config" , "globals" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/base", "no_destructor"] , ["absl/base", "raw_logging_internal"] , ["absl/cleanup", "cleanup"] , ["absl/log", "globals"] , ["absl/log", "log_entry"] , ["absl/log", "log_sink"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] , ["absl/types", "span"] ] } , "nullguard": { "type": "export" , "target": "nullguard (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "nullguard (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["nullguard"] , "stage": ["absl", "log", "internal"] , "srcs": ["nullguard.cc"] , "hdrs": ["nullguard.h"] , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] } , "nullstream": { "type": "export" , "target": "nullstream (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "nullstream (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["nullstream"] , "stage": ["absl", "log", "internal"] , "hdrs": ["nullstream.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] , ["absl/strings", "strings"] ] } , "strip": { "type": "export" , "target": "strip (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "strip (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["strip"] , "stage": ["absl", "log", "internal"] , "hdrs": ["strip.h"] , "deps": [ "log_message" , "nullstream" , ["absl/base", "core_headers"] , ["absl/base", "log_severity"] ] } , "voidify": { "type": "export" , "target": "voidify (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "voidify (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["voidify"] , "stage": ["absl", "log", "internal"] , "hdrs": ["voidify.h"] , "deps": [["absl/base", "config"]] } , "proto": { "type": ["@", "rules", "CC", "library"] , "name": ["proto"] , "stage": ["absl", "log", "internal"] , "srcs": ["proto.cc"] , "hdrs": ["proto.h"] , "deps": [ ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/strings", "strings"] , ["absl/types", "span"] ] } , "fnmatch": { "type": "export" , "target": "fnmatch (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "fnmatch (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["fnmatch"] , "stage": ["absl", "log", "internal"] , "hdrs": ["fnmatch.h"] , "srcs": ["fnmatch.cc"] , "deps": [["absl/base", "config"], ["absl/strings", "strings"]] } , "vlog_config": { "type": "export" , "target": "vlog_config (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "vlog_config (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["vlog_config"] , "stage": ["absl", "log", "internal"] , "hdrs": ["vlog_config.h"] , "srcs": ["vlog_config.cc"] , "deps": [ "fnmatch" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "no_destructor"] , ["absl/memory", "memory"] , ["absl/strings", "strings"] , ["absl/synchronization", "synchronization"] , ["absl/types", "optional"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/memory/000077500000000000000000000000001516554100600241355ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/memory/TARGETS.absl000066400000000000000000000010461516554100600261120ustar00rootroot00000000000000{ "memory": { "type": "export" , "target": "memory (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "memory (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["memory"] , "stage": ["absl", "memory"] , "hdrs": ["memory.h"] , "deps": [["absl/base", "core_headers"], ["absl/meta", "type_traits"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/meta/000077500000000000000000000000001516554100600235535ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/meta/TARGETS.absl000066400000000000000000000010701516554100600255250ustar00rootroot00000000000000{ "type_traits": { "type": "export" , "target": "type_traits (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "type_traits (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["type_traits"] , "stage": ["absl", "meta"] , "hdrs": ["type_traits.h"] , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/numeric/000077500000000000000000000000001516554100600242675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/numeric/TARGETS.absl000066400000000000000000000034571516554100600262540ustar00rootroot00000000000000{ "int128": { "type": "export" , "target": "int128 (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "int128 (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["int128"] , "stage": ["absl", "numeric"] , "hdrs": ["int128.h", "int128_have_intrinsic.inc", "int128_no_intrinsic.inc"] , "srcs": ["int128.cc"] , "deps": [ "bits" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/types", "compare"] ] } , "bits": { "type": "export" , "target": "bits (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "bits (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["bits"] , "stage": ["absl", "numeric"] , "hdrs": ["bits.h", "internal/bits.h"] , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] } , "numeric_representation": { "type": "export" , "target": "numeric_representation (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "numeric_representation (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["numeric_representation"] , "stage": ["absl", "numeric"] , "hdrs": ["internal/representation.h"] , "deps": [["absl/base", "config"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/profiling/000077500000000000000000000000001516554100600246165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/profiling/TARGETS.absl000066400000000000000000000025151516554100600265750ustar00rootroot00000000000000{ "exponential_biased": { "type": "export" , "target": "exponential_biased (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "exponential_biased (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["exponential_biased"] , "stage": ["absl", "profiling"] , "hdrs": ["internal/exponential_biased.h"] , "srcs": ["internal/exponential_biased.cc"] , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] } , "sample_recorder": { "type": "export" , "target": "sample_recorder (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "sample_recorder (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["sample_recorder"] , "stage": ["absl", "profiling"] , "hdrs": ["internal/sample_recorder.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/synchronization", "synchronization"] , ["absl/time", "time"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/random/000077500000000000000000000000001516554100600241055ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/random/TARGETS.absl000066400000000000000000000323601516554100600260650ustar00rootroot00000000000000{ "random": { "type": "export" , "target": "random (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" ] } , "random (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["random"] , "stage": ["absl", "random"] , "hdrs": ["random.h"] , "deps": [ "distributions" , "internal_nonsecure_base" , "internal_pcg_engine" , "internal_pool_urbg" , "internal_randen_engine" , "seed_sequences" ] } , "distributions": { "type": "export" , "target": "distributions (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" ] } , "distributions (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["distributions"] , "stage": ["absl", "random"] , "hdrs": [ "bernoulli_distribution.h" , "beta_distribution.h" , "discrete_distribution.h" , "distributions.h" , "exponential_distribution.h" , "gaussian_distribution.h" , "log_uniform_int_distribution.h" , "poisson_distribution.h" , "uniform_int_distribution.h" , "uniform_real_distribution.h" , "zipf_distribution.h" ] , "srcs": ["discrete_distribution.cc", "gaussian_distribution.cc"] , "deps": [ "internal_distribution_caller" , "internal_fast_uniform_bits" , "internal_fastmath" , "internal_generate_real" , "internal_iostream_state_saver" , "internal_traits" , "internal_uniform_helper" , "internal_wide_multiply" , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/strings", "strings"] ] } , "bit_gen_ref": { "type": "export" , "target": "bit_gen_ref (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "bit_gen_ref (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["bit_gen_ref"] , "stage": ["absl", "random"] , "hdrs": ["bit_gen_ref.h"] , "deps": [ "internal_pool_urbg" , "internal_salted_seed_seq" , "internal_seed_material" , "random" , ["absl/base", "core_headers"] , ["absl/base", "fast_type_id"] , ["absl/types", "span"] ] } , "internal_nonsecure_base": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_nonsecure_base"] , "stage": ["absl", "random"] , "hdrs": ["internal/nonsecure_base.h"] , "deps": [ "internal_pool_urbg" , "internal_salted_seed_seq" , "internal_seed_material" , ["absl/base", "core_headers"] , ["absl/container", "inlined_vector"] , ["absl/meta", "type_traits"] , ["absl/types", "span"] ] } , "internal_pcg_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_pcg_engine"] , "stage": ["absl", "random"] , "hdrs": ["internal/pcg_engine.h"] , "deps": [ "internal_fastmath" , "internal_iostream_state_saver" , ["absl/base", "config"] , ["absl/meta", "type_traits"] , ["absl/numeric", "int128"] ] } , "internal_pool_urbg": { "type": "export" , "target": "internal_pool_urbg (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal_pool_urbg (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_pool_urbg"] , "stage": ["absl", "random"] , "hdrs": ["internal/pool_urbg.h"] , "srcs": ["internal/pool_urbg.cc"] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] , "deps": [ "internal_randen" , "internal_seed_material" , "internal_traits" , "seed_gen_exception" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "raw_logging_internal"] , ["absl/types", "span"] ] } , "internal_randen_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_engine"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_engine.h"] , "deps": [ "internal_iostream_state_saver" , "internal_randen" , ["absl/base", "endian"] , ["absl/meta", "type_traits"] ] } , "seed_sequences": { "type": ["@", "rules", "CC", "library"] , "name": ["seed_sequences"] , "stage": ["absl", "random"] , "hdrs": ["seed_sequences.h"] , "srcs": ["seed_sequences.cc"] , "deps": [ "internal_pool_urbg" , "internal_salted_seed_seq" , "internal_seed_material" , "seed_gen_exception" , ["absl/base", "config"] , ["absl/base", "nullability"] , ["absl/strings", "string_view"] , ["absl/types", "span"] ] } , "internal_generate_real": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_generate_real"] , "stage": ["absl", "random"] , "hdrs": ["internal/generate_real.h"] , "deps": [ "internal_fastmath" , "internal_traits" , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] ] } , "internal_distribution_caller": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_distribution_caller"] , "stage": ["absl", "random"] , "hdrs": ["internal/distribution_caller.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "fast_type_id"] , ["absl/utility", "utility"] ] } , "internal_fast_uniform_bits": { "type": "export" , "target": "internal_fast_uniform_bits (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal_fast_uniform_bits (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_fast_uniform_bits"] , "stage": ["absl", "random"] , "hdrs": ["internal/fast_uniform_bits.h"] , "deps": ["internal_traits", ["absl/base", "config"], ["absl/meta", "type_traits"]] } , "internal_fastmath": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_fastmath"] , "stage": ["absl", "random"] , "hdrs": ["internal/fastmath.h"] , "deps": [["absl/numeric", "bits"]] } , "internal_iostream_state_saver": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_iostream_state_saver"] , "stage": ["absl", "random"] , "hdrs": ["internal/iostream_state_saver.h"] , "deps": [["absl/meta", "type_traits"], ["absl/numeric", "int128"]] } , "internal_traits": { "type": "export" , "target": "internal_traits (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal_traits (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_traits"] , "stage": ["absl", "random"] , "hdrs": ["internal/traits.h"] , "deps": [ ["absl/base", "config"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] ] } , "internal_uniform_helper": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_uniform_helper"] , "stage": ["absl", "random"] , "hdrs": ["internal/uniform_helper.h"] , "deps": [ "internal_traits" , ["absl/base", "config"] , ["absl/meta", "type_traits"] , ["absl/numeric", "int128"] ] } , "internal_wide_multiply": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_wide_multiply"] , "stage": ["absl", "random"] , "hdrs": ["internal/wide_multiply.h"] , "deps": [ "internal_traits" , ["absl/base", "config"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] ] } , "internal_salted_seed_seq": { "type": "export" , "target": "internal_salted_seed_seq (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal_salted_seed_seq (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_salted_seed_seq"] , "stage": ["absl", "random"] , "hdrs": ["internal/salted_seed_seq.h"] , "deps": [ "internal_seed_material" , ["absl/container", "inlined_vector"] , ["absl/meta", "type_traits"] , ["absl/types", "optional"] , ["absl/types", "span"] ] } , "internal_seed_material": { "type": "export" , "target": "internal_seed_material (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal_seed_material (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "OS"] , "name": ["internal_seed_material"] , "stage": ["absl", "random"] , "hdrs": ["internal/seed_material.h"] , "srcs": ["internal/seed_material.cc"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["windows", ["-lbcrypt"]]] } } , "deps": { "type": "++" , "$1": [ [ "internal_fast_uniform_bits" , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "raw_logging_internal"] , ["absl/strings", "strings"] , ["absl/types", "optional"] , ["absl/types", "span"] ] , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["windows", [["@", "bcrypt", "", "bcrypt"]]]] } } ] } } , "internal_randen": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen.h"] , "srcs": ["internal/randen.cc"] , "deps": [ "internal_platform" , "internal_randen_hwaes" , "internal_randen_slow" , ["absl/base", "raw_logging_internal"] ] } , "seed_gen_exception": { "type": ["@", "rules", "CC", "library"] , "name": ["seed_gen_exception"] , "stage": ["absl", "random"] , "hdrs": ["seed_gen_exception.h"] , "srcs": ["seed_gen_exception.cc"] , "deps": [["absl/base", "config"]] } , "internal_platform": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_platform"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_traits.h", "internal/platform.h"] , "srcs": ["internal/randen_round_keys.cc"] , "deps": [["absl/base", "config"]] } , "internal_randen_hwaes": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_hwaes"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_detect.h", "internal/randen_hwaes.h"] , "srcs": ["internal/randen_detect.cc"] , "deps": [ "internal_platform" , "internal_randen_hwaes_impl" , ["absl/base", "config"] ] } , "internal_randen_slow": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_slow"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_slow.h"] , "srcs": ["internal/randen_slow.cc"] , "deps": [ "internal_platform" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/numeric", "int128"] ] } , "internal_randen_hwaes_impl": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_hwaes_impl"] , "arguments_config": ["ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "stage": ["absl", "random"] , "private-hdrs": ["internal/randen_hwaes.h"] , "srcs": ["internal/randen_hwaes.cc"] , "private-cflags": { "type": "case*" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": [ ["arm", ["-mfpu=neon"]] , ["arm64", ["-march=armv8-a+crypto"]] , [ "x86_64" , { "type": "case*" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "case": [["msvc", []]] , "default": ["-maes", "-msse4.1"] } ] ] } , "deps": [ "internal_platform" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/numeric", "int128"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/status/000077500000000000000000000000001516554100600241505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/status/TARGETS.absl000066400000000000000000000042561516554100600261330ustar00rootroot00000000000000{ "status": { "type": "export" , "target": "status (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "status (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["status"] , "stage": ["absl", "status"] , "hdrs": ["status.h", "status_payload_printer.h", "internal/status_internal.h"] , "srcs": ["internal/status_internal.cc", "status.cc", "status_payload_printer.cc"] , "deps": [ ["absl/base", "atomic_hook"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "no_destructor"] , ["absl/base", "nullability"] , ["absl/base", "raw_logging_internal"] , ["absl/base", "strerror"] , ["absl/container", "inlined_vector"] , ["absl/debugging", "stacktrace"] , ["absl/debugging", "symbolize"] , ["absl/functional", "function_ref"] , ["absl/memory", "memory"] , ["absl/strings", "cord"] , ["absl/strings", "str_format"] , ["absl/strings", "strings"] , ["absl/types", "optional"] , ["absl/types", "span"] ] } , "statusor": { "type": "export" , "target": "statusor (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "statusor (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["statusor"] , "stage": ["absl", "status"] , "hdrs": ["statusor.h", "internal/statusor_internal.h"] , "srcs": ["statusor.cc"] , "deps": [ "status" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/base", "raw_logging_internal"] , ["absl/meta", "type_traits"] , ["absl/strings", "has_ostream_operator"] , ["absl/strings", "str_format"] , ["absl/strings", "strings"] , ["absl/types", "variant"] , ["absl/utility", "utility"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/strings/000077500000000000000000000000001516554100600243165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/strings/TARGETS.absl000066400000000000000000000265111516554100600262770ustar00rootroot00000000000000{ "string_view": { "type": "export" , "target": "string_view (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "string_view (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["string_view"] , "stage": ["absl", "strings"] , "hdrs": ["string_view.h"] , "srcs": ["string_view.cc"] , "deps": [ ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/base", "throw_delegate"] ] } , "strings": { "type": "export" , "target": "strings (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "strings (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["strings"] , "stage": ["absl", "strings"] , "hdrs": [ "ascii.h" , "charconv.h" , "escaping.h" , "has_absl_stringify.h" , "internal/damerau_levenshtein_distance.h" , "internal/string_constant.h" , "match.h" , "numbers.h" , "str_cat.h" , "str_join.h" , "str_replace.h" , "str_split.h" , "string_view.h" , "strip.h" , "substitute.h" , "internal/stringify_sink.h" , "internal/str_join_internal.h" , "internal/str_split_internal.h" , "internal/stl_type_traits.h" ] , "srcs": [ "ascii.cc" , "charconv.cc" , "escaping.cc" , "internal/charconv_bigint.cc" , "internal/charconv_parse.cc" , "internal/damerau_levenshtein_distance.cc" , "internal/memutil.cc" , "internal/stringify_sink.cc" , "match.cc" , "numbers.cc" , "str_cat.cc" , "str_replace.cc" , "str_split.cc" , "substitute.cc" ] , "private-hdrs": [ "internal/charconv_bigint.h" , "internal/charconv_parse.h" , "internal/memutil.h" ] , "deps": [ "charset" , "internal" , "string_view" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "nullability"] , ["absl/base", "raw_logging_internal"] , ["absl/base", "throw_delegate"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] ] } , "internal": { "type": "export" , "target": "internal (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["internal"] , "stage": ["absl", "strings"] , "hdrs": [ "internal/escaping.h" , "internal/ostringstream.h" , "internal/resize_uninitialized.h" , "internal/utf8.h" ] , "srcs": ["internal/escaping.cc", "internal/ostringstream.cc", "internal/utf8.cc"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "raw_logging_internal"] , ["absl/meta", "type_traits"] ] } , "cord": { "type": "export" , "target": "cord (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "cord (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["cord"] , "stage": ["absl", "strings"] , "hdrs": ["cord.h", "cord_buffer.h", "cord_analysis.h"] , "srcs": ["cord.cc", "cord_analysis.cc", "cord_buffer.cc"] , "deps": [ "cord_internal" , "cordz_functions" , "cordz_info" , "cordz_statistics" , "cordz_update_scope" , "cordz_update_tracker" , "internal" , "strings" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "nullability"] , ["absl/base", "raw_logging_internal"] , ["absl/container", "inlined_vector"] , ["absl/crc", "crc32c"] , ["absl/crc", "crc_cord_state"] , ["absl/functional", "function_ref"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/types", "compare"] , ["absl/types", "optional"] , ["absl/types", "span"] ] } , "cord_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["cord_internal"] , "stage": ["absl", "strings"] , "hdrs": [ "internal/cord_data_edge.h" , "internal/cord_internal.h" , "internal/cord_rep_btree.h" , "internal/cord_rep_btree_navigator.h" , "internal/cord_rep_btree_reader.h" , "internal/cord_rep_crc.h" , "internal/cord_rep_consume.h" , "internal/cord_rep_flat.h" ] , "srcs": [ "internal/cord_internal.cc" , "internal/cord_rep_btree.cc" , "internal/cord_rep_btree_navigator.cc" , "internal/cord_rep_btree_reader.cc" , "internal/cord_rep_crc.cc" , "internal/cord_rep_consume.cc" ] , "deps": [ "strings" , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "raw_logging_internal"] , ["absl/base", "throw_delegate"] , ["absl/container", "compressed_tuple"] , ["absl/container", "container_memory"] , ["absl/container", "inlined_vector"] , ["absl/container", "layout"] , ["absl/crc", "crc_cord_state"] , ["absl/functional", "function_ref"] , ["absl/meta", "type_traits"] , ["absl/types", "span"] ] } , "cordz_functions": { "type": ["@", "rules", "CC", "library"] , "name": ["cordz_functions"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_functions.h"] , "srcs": ["internal/cordz_functions.cc"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] , ["absl/profiling", "exponential_biased"] ] } , "cordz_info": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["cordz_info"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_info.h"] , "srcs": ["internal/cordz_info.cc"] , "private-ldflags": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": [ "cord_internal" , "cordz_functions" , "cordz_handle" , "cordz_statistics" , "cordz_update_tracker" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] , ["absl/container", "inlined_vector"] , ["absl/debugging", "stacktrace"] , ["absl/synchronization", "synchronization"] , ["absl/time", "time"] , ["absl/types", "span"] ] } , "cordz_update_scope": { "type": ["@", "rules", "CC", "library"] , "name": ["cordz_update_scope"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_update_scope.h"] , "deps": [ "cord_internal" , "cordz_info" , "cordz_update_tracker" , ["absl/base", "config"] , ["absl/base", "core_headers"] ] } , "cordz_update_tracker": { "type": ["@", "rules", "CC", "library"] , "name": ["cordz_update_tracker"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_update_tracker.h"] , "deps": [["absl/base", "config"]] } , "cordz_handle": { "type": ["@", "rules", "CC", "library"] , "name": ["cordz_handle"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_handle.h"] , "srcs": ["internal/cordz_handle.cc"] , "deps": [ ["absl/base", "config"] , ["absl/base", "no_destructor"] , ["absl/base", "raw_logging_internal"] , ["absl/synchronization", "synchronization"] ] } , "cordz_statistics": { "type": ["@", "rules", "CC", "library"] , "name": ["cordz_statistics"] , "stage": ["absl", "strings"] , "hdrs": ["internal/cordz_statistics.h"] , "deps": ["cordz_update_tracker", ["absl/base", "config"]] } , "str_format": { "type": "export" , "target": "str_format (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "str_format (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["str_format"] , "stage": ["absl", "strings"] , "hdrs": ["str_format.h"] , "deps": [ "str_format_internal" , "string_view" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/types", "span"] ] } , "str_format_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["str_format_internal"] , "stage": ["absl", "strings"] , "hdrs": [ "internal/str_format/arg.h" , "internal/str_format/bind.h" , "internal/str_format/checker.h" , "internal/str_format/constexpr_parser.h" , "internal/str_format/extension.h" , "internal/str_format/float_conversion.h" , "internal/str_format/output.h" , "internal/str_format/parser.h" ] , "srcs": [ "internal/str_format/arg.cc" , "internal/str_format/bind.cc" , "internal/str_format/extension.cc" , "internal/str_format/float_conversion.cc" , "internal/str_format/output.cc" , "internal/str_format/parser.cc" ] , "deps": [ "strings" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/container", "fixed_array"] , ["absl/container", "inlined_vector"] , ["absl/functional", "function_ref"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] , ["absl/numeric", "numeric_representation"] , ["absl/types", "optional"] , ["absl/types", "span"] , ["absl/utility", "utility"] ] } , "has_ostream_operator": { "type": "export" , "target": "has_ostream_operator (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "has_ostream_operator (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["has_ostream_operator"] , "stage": ["absl", "strings"] , "hdrs": ["has_ostream_operator.h"] , "deps": [["absl/base", "config"]] } , "charset": { "type": "export" , "target": "charset (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "charset (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["charset"] , "stage": ["absl", "strings"] , "hdrs": ["charset.h"] , "deps": ["string_view", ["absl/base", "core_headers"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/synchronization/000077500000000000000000000000001516554100600260665ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/synchronization/TARGETS.absl000066400000000000000000000060611516554100600300450ustar00rootroot00000000000000{ "synchronization": { "type": "export" , "target": "synchronization (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "synchronization (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["synchronization"] , "stage": ["absl", "synchronization"] , "hdrs": [ "barrier.h" , "blocking_counter.h" , "internal/create_thread_identity.h" , "internal/futex.h" , "internal/futex_waiter.h" , "internal/per_thread_sem.h" , "internal/pthread_waiter.h" , "internal/sem_waiter.h" , "internal/stdcpp_waiter.h" , "internal/waiter.h" , "internal/waiter_base.h" , "internal/win32_waiter.h" , "mutex.h" , "notification.h" ] , "srcs": [ "barrier.cc" , "blocking_counter.cc" , "internal/create_thread_identity.cc" , "internal/futex_waiter.cc" , "internal/per_thread_sem.cc" , "internal/pthread_waiter.cc" , "internal/sem_waiter.cc" , "internal/stdcpp_waiter.cc" , "internal/waiter_base.cc" , "internal/win32_waiter.cc" , "mutex.cc" , "notification.cc" ] , "private-ldflags": { "type": "++" , "$1": [ ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } , "deps": [ "graphcycles_internal" , "kernel_timeout_internal" , ["absl/base", "atomic_hook"] , ["absl/base", "base"] , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "malloc_internal"] , ["absl/base", "raw_logging_internal"] , ["absl/debugging", "stacktrace"] , ["absl/debugging", "symbolize"] , ["absl/time", "time"] ] } , "graphcycles_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["graphcycles_internal"] , "stage": ["absl", "synchronization"] , "hdrs": ["internal/graphcycles.h"] , "srcs": ["internal/graphcycles.cc"] , "deps": [ ["absl/base", "base"] , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "malloc_internal"] , ["absl/base", "raw_logging_internal"] ] } , "kernel_timeout_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["kernel_timeout_internal"] , "stage": ["absl", "synchronization"] , "srcs": ["internal/kernel_timeout.cc"] , "hdrs": ["internal/kernel_timeout.h"] , "deps": [ ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] , ["absl/time", "time"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/time/000077500000000000000000000000001516554100600235635ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/time/TARGETS.absl000066400000000000000000000051361516554100600255440ustar00rootroot00000000000000{ "time": { "type": "export" , "target": "time (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "time (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["time"] , "stage": ["absl", "time"] , "hdrs": ["civil_time.h", "clock.h", "time.h"] , "srcs": ["civil_time.cc", "clock.cc", "duration.cc", "format.cc", "time.cc"] , "private-hdrs": [ "internal/get_current_time_chrono.inc" , "internal/get_current_time_posix.inc" ] , "deps": [ "civil_time" , "time_zone" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "raw_logging_internal"] , ["absl/numeric", "int128"] , ["absl/strings", "strings"] , ["absl/types", "optional"] ] } , "civil_time": { "type": ["@", "rules", "CC", "library"] , "name": ["civil_time"] , "stage": ["absl", "time"] , "hdrs": [ "internal/cctz/include/cctz/civil_time.h" , "internal/cctz/include/cctz/civil_time_detail.h" ] , "srcs": ["internal/cctz/src/civil_time_detail.cc"] , "deps": [["absl/base", "config"]] } , "time_zone": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["time_zone"] , "stage": ["absl", "time"] , "hdrs": [ "internal/cctz/include/cctz/time_zone.h" , "internal/cctz/include/cctz/zone_info_source.h" ] , "srcs": [ "internal/cctz/src/time_zone_fixed.cc" , "internal/cctz/src/time_zone_format.cc" , "internal/cctz/src/time_zone_if.cc" , "internal/cctz/src/time_zone_impl.cc" , "internal/cctz/src/time_zone_info.cc" , "internal/cctz/src/time_zone_libc.cc" , "internal/cctz/src/time_zone_lookup.cc" , "internal/cctz/src/time_zone_posix.cc" , "internal/cctz/src/zone_info_source.cc" ] , "private-hdrs": [ "internal/cctz/src/time_zone_fixed.h" , "internal/cctz/src/time_zone_if.h" , "internal/cctz/src/time_zone_impl.h" , "internal/cctz/src/time_zone_info.h" , "internal/cctz/src/time_zone_libc.h" , "internal/cctz/src/time_zone_posix.h" , "internal/cctz/src/tzfile.h" ] , "private-ldflags": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["civil_time", ["absl/base", "config"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/types/000077500000000000000000000000001516554100600237715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/types/TARGETS.absl000066400000000000000000000064551516554100600257570ustar00rootroot00000000000000{ "optional": { "type": "export" , "target": "optional (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "optional (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["optional"] , "stage": ["absl", "types"] , "hdrs": ["optional.h", "internal/optional.h"] , "deps": [ "bad_optional_access" , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/memory", "memory"] , ["absl/meta", "type_traits"] , ["absl/utility", "utility"] ] } , "bad_optional_access": { "type": ["@", "rules", "CC", "library"] , "name": ["bad_optional_access"] , "stage": ["absl", "types"] , "hdrs": ["bad_optional_access.h"] , "srcs": ["bad_optional_access.cc"] , "deps": [["absl/base", "config"], ["absl/base", "raw_logging_internal"]] } , "variant": { "type": "export" , "target": "variant (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "variant (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["variant"] , "stage": ["absl", "types"] , "hdrs": ["variant.h", "internal/variant.h"] , "deps": [ "bad_variant_access" , ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] , ["absl/utility", "utility"] ] } , "bad_variant_access": { "type": ["@", "rules", "CC", "library"] , "name": ["bad_variant_access"] , "stage": ["absl", "types"] , "hdrs": ["bad_variant_access.h"] , "srcs": ["bad_variant_access.cc"] , "deps": [["absl/base", "config"], ["absl/base", "raw_logging_internal"]] } , "span": { "type": "export" , "target": "span (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "span (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["span"] , "stage": ["absl", "types"] , "hdrs": ["span.h", "internal/span.h"] , "deps": [ ["absl/algorithm", "algorithm"] , ["absl/base", "core_headers"] , ["absl/base", "nullability"] , ["absl/base", "throw_delegate"] , ["absl/meta", "type_traits"] ] } , "compare": { "type": "export" , "target": "compare (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "compare (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["compare"] , "stage": ["absl", "types"] , "hdrs": ["compare.h"] , "deps": [ ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/absl/utility/000077500000000000000000000000001516554100600243305ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/absl/utility/TARGETS.absl000066400000000000000000000022021516554100600263000ustar00rootroot00000000000000{ "utility": { "type": "export" , "target": "utility (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "utility (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["utility"] , "stage": ["absl", "utility"] , "hdrs": ["utility.h"] , "deps": [ ["absl/base", "base_internal"] , ["absl/base", "config"] , ["absl/meta", "type_traits"] ] } , "if_constexpr": { "type": "export" , "target": "if_constexpr (unexported)" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ARCH" , "CC" , "CFLAGS" , "CXX" , "CXXFLAGS" , "DEBUG" , "ENV" , "HOST_ARCH" , "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" ] } , "if_constexpr (unexported)": { "type": ["@", "rules", "CC", "library"] , "name": ["if_constexpr"] , "stage": ["absl", "utility"] , "hdrs": ["internal/if_constexpr.h"] , "deps": [["absl/base", "config"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/deps/000077500000000000000000000000001516554100600226375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/deps/http-parser/000077500000000000000000000000001516554100600251105ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/deps/http-parser/TARGETS.git2000066400000000000000000000010621516554100600270070ustar00rootroot00000000000000{ "git2_http_parser": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["TOOLCHAIN_CONFIG"] , "name": ["git2_http_parser"] , "pure C": ["yes"] , "srcs": ["http_parser.c"] , "hdrs": ["http_parser.h"] , "private-cflags": { "type": "case" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } , "case": {"gnu": ["-Wimplicit-fallthrough=1"]} } } } just-buildsystem-justbuild-b1fb5fa/etc/import/deps/pcre/000077500000000000000000000000001516554100600235705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/deps/pcre/TARGETS.git2000066400000000000000000000131131516554100600254670ustar00rootroot00000000000000{ "config.h": { "type": "configure" , "target": "config_header" , "config": { "type": "let*" , "bindings": [ ["SUPPORT_PCRE8", true] , ["BSR_ANYCRLF", false] , ["NO_RECURSE", true] , ["PCRE_LINK_SIZE", "2"] , ["PCRE_PARENS_NEST_LIMIT", "250"] , ["PCRE_MATCH_LIMIT", "10000000"] , ["PCRE_MATCH_LIMIT_RECURSION", "MATCH_LIMIT"] , ["PCRE_NEWLINE", "LF"] , ["PCRE_POSIX_MALLOC_THRESHOLD", "10"] , ["sys_hdrs", ["stdlib.h", "string.h", "strings.h"]] , [ "defines" , [ ["MAX_NAME_SIZE", "32"] , ["MAX_NAME_COUNT", "10000"] , [ "NEWLINE" , { "type": "case" , "expr": {"type": "var", "name": "PCRE_NEWLINE"} , "case": { "LF": "10" , "CR": "13" , "CRLF": "3338" , "ANY": "-1" , "ANYCRLF": "-2" } , "default": { "type": "fail" , "msg": [ "The PCRE_NEWLINE variable must be set to one of the following values:" , ["LF", "CR", "CRLF", "ANY", "ANYCRLF"] ] } } ] , [ "POSIX_MALLOC_THRESHOLD" , {"type": "var", "name": "PCRE_POSIX_MALLOC_THRESHOLD"} ] , ["LINK_SIZE", {"type": "var", "name": "PCRE_LINK_SIZE"}] , [ "PARENS_NEST_LIMIT" , {"type": "var", "name": "PCRE_PARENS_NEST_LIMIT"} ] , ["MATCH_LIMIT", {"type": "var", "name": "PCRE_MATCH_LIMIT"}] , [ "MATCH_LIMIT_RECURSION" , {"type": "var", "name": "PCRE_MATCH_LIMIT_RECURSION"} ] , [ "PCREGREP_BUFSIZE" , {"type": "var", "name": "PCREGREP_BUFSIZE", "default": ""} ] ] ] , [ "defines1" , [ ["PCRE_STATIC", {"type": "var", "name": "PCRE_STATIC"}] , ["SUPPORT_PCRE8", {"type": "var", "name": "SUPPORT_PCRE8"}] , ["SUPPORT_PCRE16", {"type": "var", "name": "SUPPORT_PCRE16"}] , ["SUPPORT_PCRE32", {"type": "var", "name": "SUPPORT_PCRE32"}] , ["SUPPORT_JIT", {"type": "var", "name": "SUPPORT_JIT"}] , [ "SUPPORT_PCREGREP_JIT" , {"type": "var", "name": "SUPPORT_PCREGREP_JIT"} ] , ["SUPPORT_UTF", {"type": "var", "name": "SUPPORT_UTF"}] , ["SUPPORT_UCP", {"type": "var", "name": "SUPPORT_UCP"}] , ["EBCDIC", {"type": "var", "name": "EBCDIC"}] , ["EBCDIC_NL25", {"type": "var", "name": "EBCDIC_NL25"}] , ["BSR_ANYCRLF", {"type": "var", "name": "BSR_ANYCRLF"}] , ["NO_RECURSE", {"type": "var", "name": "NO_RECURSE"}] , ["SUPPORT_LIBBZ2", {"type": "var", "name": "SUPPORT_LIBBZ2"}] , ["SUPPORT_LIBZ", {"type": "var", "name": "SUPPORT_LIBZ"}] , ["SUPPORT_LIBEDIT", {"type": "var", "name": "SUPPORT_LIBEDIT"}] , [ "SUPPORT_LIBREADLINE" , {"type": "var", "name": "SUPPORT_LIBREADLINE"} ] , ["SUPPORT_VALGRIND", {"type": "var", "name": "SUPPORT_VALGRIND"}] , ["SUPPORT_GCOV", {"type": "var", "name": "SUPPORT_GCOV"}] ] ] , [ "have_cfile" , [ ["HAVE_DIRENT_H", "dirent.h"] , ["HAVE_STDINT_H", "stdint.h"] , ["HAVE_INTTYPES_H", "inttypes.h"] , ["HAVE_SYS_STAT_H", "sys/stat.h"] , ["HAVE_SYS_TYPES_H", "sys/types.h"] , ["HAVE_UNISTD_H", "unistd.h"] , ["HAVE_WINDOWS_H", "windows.h"] ] ] , [ "have_ctype" , [ ["HAVE_LONG_LONG", "long long"] , ["HAVE_UNSIGNED_LONG_LONG", "unsigned long long"] ] ] , [ "have_csymbol" , [ ["HAVE_BCOPY", ["bcopy", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_MEMMOVE", ["memmove", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_STRERROR", ["bcopy", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_STRTOLL", ["strtoll", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_STRTOQ", ["strtoq", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE__STRTOI64" , ["_strtoi64", {"type": "var", "name": "sys_hdrs"}] ] ] ] ] , "body": { "type": "env" , "vars": ["defines", "defines1", "have_cfile", "have_ctype", "have_csymbol"] } } } , "config_header": { "type": ["@", "rules", "CC/auto", "config"] , "name": ["config.h"] , "guard": ["INCLUDE_DEPS_PCRE_CONFIG_H"] } , "git2_pcre": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS"] , "name": ["git2_pcre"] , "pure C": ["yes"] , "private-defines": { "type": "++" , "$1": [ ["HAVE_CONFIG_H"] , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": {"windows": ["_CRT_SECURE_NO_DEPRECATE", "_CRT_SECURE_NO_WARNINGS"]} } ] } , "private-cflags": ["-Wno-unused-function", "-Wno-implicit-fallthrough"] , "srcs": [ "pcre_byte_order.c" , "pcre_chartables.c" , "pcre_compile.c" , "pcre_config.c" , "pcre_dfa_exec.c" , "pcre_exec.c" , "pcre_fullinfo.c" , "pcre_get.c" , "pcre_globals.c" , "pcre_jit_compile.c" , "pcre_maketables.c" , "pcre_newline.c" , "pcre_ord2utf8.c" , "pcre_refcount.c" , "pcre_string_utils.c" , "pcre_study.c" , "pcre_tables.c" , "pcre_ucd.c" , "pcre_valid_utf8.c" , "pcre_version.c" , "pcre_xclass.c" , "pcreposix.c" ] , "hdrs": ["config.h", "pcre.h", "pcre_internal.h", "ucp.h", "pcreposix.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/deps/xdiff/000077500000000000000000000000001516554100600237375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/deps/xdiff/TARGETS.git2000066400000000000000000000002621516554100600256370ustar00rootroot00000000000000{ "srcs": {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.c"]]} , "hdrs": {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.h"]]} } just-buildsystem-justbuild-b1fb5fa/etc/import/include/000077500000000000000000000000001516554100600233275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/CLI/000077500000000000000000000000001516554100600237365ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/CLI/TARGETS.cli11000066400000000000000000000001741516554100600257040ustar00rootroot00000000000000{ "cli11_headers": { "type": "tree" , "name": "CLI" , "deps": [["GLOB", null, "*.hpp"], ["TREE", null, "impl"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/TARGETS.cares000066400000000000000000000003041516554100600254540ustar00rootroot00000000000000{ "ares_include_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "ares.h" , "ares_dns.h" , "ares_nameser.h" , "ares_rules.h" , "ares_version.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/TARGETS.curl000066400000000000000000000000031516554100600253200ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/import/include/TARGETS.git2000066400000000000000000000001731516554100600252300ustar00rootroot00000000000000{ "git2_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["git2.h", ["./", "git2", "TREE"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/curl/000077500000000000000000000000001516554100600242745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/curl/TARGETS.curl000066400000000000000000000004611516554100600262750ustar00rootroot00000000000000{ "curl_public_headers": { "type": "tree" , "name": "curl" , "deps": [ "curl.h" , "curlver.h" , "easy.h" , "header.h" , "mprintf.h" , "multi.h" , "options.h" , "stdcheaders.h" , "system.h" , "typecheck-gcc.h" , "urlapi.h" , "websockets.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/fmt/000077500000000000000000000000001516554100600241155ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/fmt/TARGETS.fmt000066400000000000000000000001641516554100600257370ustar00rootroot00000000000000{ "hdrs": { "type": ["@", "rules", "data", "staged"] , "stage": ["fmt"] , "srcs": [["TREE", null, "."]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/git2/000077500000000000000000000000001516554100600241745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/git2/TARGETS.git2000066400000000000000000000034271516554100600261020ustar00rootroot00000000000000{ "TREE": { "type": "tree" , "name": "git2" , "deps": [ "annotated_commit.h" , "apply.h" , "attr.h" , "blame.h" , "blob.h" , "branch.h" , "buffer.h" , "cert.h" , "checkout.h" , "cherrypick.h" , "clone.h" , "commit.h" , "common.h" , "config.h" , "cred_helpers.h" , "credential.h" , "credential_helpers.h" , "deprecated.h" , "describe.h" , "diff.h" , "email.h" , "errors.h" , "filter.h" , "global.h" , "graph.h" , "ignore.h" , "index.h" , "indexer.h" , "mailmap.h" , "merge.h" , "message.h" , "net.h" , "notes.h" , "object.h" , "odb.h" , "odb_backend.h" , "oid.h" , "oidarray.h" , "pack.h" , "patch.h" , "pathspec.h" , "proxy.h" , "rebase.h" , "refdb.h" , "reflog.h" , "refs.h" , "refspec.h" , "remote.h" , "repository.h" , "reset.h" , "revert.h" , "revparse.h" , "revwalk.h" , "signature.h" , "stash.h" , "status.h" , "stdint.h" , "strarray.h" , "submodule.h" , "sys/alloc.h" , "sys/commit.h" , "sys/commit_graph.h" , "sys/config.h" , "sys/cred.h" , "sys/credential.h" , "sys/diff.h" , "sys/email.h" , "sys/filter.h" , "sys/hashsig.h" , "sys/index.h" , "sys/mempack.h" , "sys/merge.h" , "sys/midx.h" , "sys/odb_backend.h" , "sys/openssl.h" , "sys/path.h" , "sys/refdb_backend.h" , "sys/reflog.h" , "sys/refs.h" , "sys/remote.h" , "sys/repository.h" , "sys/stream.h" , "sys/transport.h" , "tag.h" , "trace.h" , "transaction.h" , "transport.h" , "tree.h" , "types.h" , "version.h" , "worktree.h" , ["src/libgit2", "experimental.h"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpc++/000077500000000000000000000000001516554100600244105ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpc++/TARGETS.grpc000066400000000000000000000053461516554100600264060ustar00rootroot00000000000000{ "grpc++_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "alarm.h" , "channel.h" , "client_context.h" , "completion_queue.h" , "create_channel.h" , "create_channel_posix.h" , "ext/health_check_service_server_builder_option.h" , "generic/async_generic_service.h" , "generic/generic_stub.h" , "grpc++.h" , "health_check_service_interface.h" , "impl/call.h" , "impl/channel_argument_option.h" , "impl/client_unary_call.h" , "impl/grpc_library.h" , "impl/method_handler_impl.h" , "impl/rpc_method.h" , "impl/rpc_service_method.h" , "impl/serialization_traits.h" , "impl/server_builder_option.h" , "impl/server_builder_plugin.h" , "impl/server_initializer.h" , "impl/service_type.h" , "security/auth_context.h" , "resource_quota.h" , "security/auth_metadata_processor.h" , "security/credentials.h" , "security/server_credentials.h" , "server.h" , "server_builder.h" , "server_context.h" , "server_posix.h" , "support/async_stream.h" , "support/async_unary_call.h" , "support/byte_buffer.h" , "support/channel_arguments.h" , "support/config.h" , "support/slice.h" , "support/status.h" , "support/status_code_enum.h" , "support/string_ref.h" , "support/stub_options.h" , "support/sync_stream.h" , "support/time.h" , "impl/codegen/async_stream.h" , "impl/codegen/async_unary_call.h" , "impl/codegen/byte_buffer.h" , "impl/codegen/call_hook.h" , "impl/codegen/call.h" , "impl/codegen/channel_interface.h" , "impl/codegen/client_context.h" , "impl/codegen/client_unary_call.h" , "impl/codegen/completion_queue_tag.h" , "impl/codegen/completion_queue.h" , "impl/codegen/config.h" , "impl/codegen/create_auth_context.h" , "impl/codegen/metadata_map.h" , "impl/codegen/method_handler_impl.h" , "impl/codegen/rpc_method.h" , "impl/codegen/rpc_service_method.h" , "impl/codegen/security/auth_context.h" , "impl/codegen/serialization_traits.h" , "impl/codegen/server_context.h" , "impl/codegen/server_interface.h" , "impl/codegen/service_type.h" , "impl/codegen/slice.h" , "impl/codegen/status_code_enum.h" , "impl/codegen/status.h" , "impl/codegen/string_ref.h" , "impl/codegen/stub_options.h" , "impl/codegen/sync_stream.h" , "impl/codegen/time.h" ] , "stage": ["grpc++"] } , "grpc++_config_proto_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["impl/codegen/config_protobuf.h"] , "stage": ["grpc++"] } , "grpc++_codegen_proto_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["impl/codegen/proto_utils.h"] , "stage": ["grpc++"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpc/000077500000000000000000000000001516554100600242625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpc/TARGETS.grpc000066400000000000000000000101121516554100600262430ustar00rootroot00000000000000{ "gpr_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "support/alloc.h" , "support/atm_gcc_atomic.h" , "support/atm_gcc_sync.h" , "support/atm_windows.h" , "support/atm.h" , "support/cpu.h" , "support/json.h" , "support/log.h" , "support/log_windows.h" , "support/metrics.h" , "support/port_platform.h" , "support/string_util.h" , "support/sync.h" , "support/sync_abseil.h" , "support/sync_custom.h" , "support/sync_generic.h" , "support/sync_posix.h" , "support/sync_windows.h" , "support/thd_id.h" , "support/time.h" , "impl/call.h" , "impl/codegen/atm.h" , "impl/codegen/atm_gcc_atomic.h" , "impl/codegen/atm_gcc_sync.h" , "impl/codegen/atm_windows.h" , "impl/codegen/fork.h" , "impl/codegen/gpr_types.h" , "impl/codegen/log.h" , "impl/codegen/port_platform.h" , "impl/codegen/sync.h" , "impl/codegen/sync_abseil.h" , "impl/codegen/sync_custom.h" , "impl/codegen/sync_generic.h" , "impl/codegen/sync_posix.h" , "impl/codegen/sync_windows.h" ] , "stage": ["grpc"] } , "grpc_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "grpc_audit_logging.h" , "grpc_crl_provider.h" , "byte_buffer.h" , "byte_buffer_reader.h" , "compression.h" , "fork.h" , "grpc.h" , "grpc_posix.h" , "grpc_security.h" , "grpc_security_constants.h" , "passive_listener.h" , "slice.h" , "slice_buffer.h" , "status.h" , "load_reporting.h" , "support/workaround_list.h" , "impl/codegen/byte_buffer.h" , "impl/codegen/byte_buffer_reader.h" , "impl/codegen/compression_types.h" , "impl/codegen/connectivity_state.h" , "impl/codegen/grpc_types.h" , "impl/codegen/propagation_bits.h" , "impl/codegen/status.h" , "impl/codegen/slice.h" , "impl/compression_types.h" , "impl/connectivity_state.h" , "impl/grpc_types.h" , "impl/propagation_bits.h" , "impl/slice_type.h" ] , "stage": ["grpc"] } , "grpc_public_event_engine_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "event_engine/endpoint_config.h" , "event_engine/event_engine.h" , "event_engine/extensible.h" , "event_engine/port.h" , "event_engine/memory_allocator.h" , "event_engine/memory_request.h" , "event_engine/internal/memory_allocator_impl.h" , "event_engine/slice.h" , "event_engine/slice_buffer.h" , "event_engine/internal/slice_cast.h" ] , "stage": ["grpc"] } , "grpc_secure_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["grpc_security.h"] , "stage": ["grpc"] } , "event_engine_memory_allocator_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "event_engine/internal/memory_allocator_impl.h" , "event_engine/memory_allocator.h" , "event_engine/memory_request.h" ] , "stage": ["grpc"] } , "slice_cast_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["event_engine/internal/slice_cast.h"] , "stage": ["grpc"] } , "slice_buffer_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["event_engine/slice.h", "event_engine/slice_buffer.h"] , "stage": ["grpc"] } , "gpr_atm_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "support/atm.h" , "support/atm_gcc_atomic.h" , "support/atm_gcc_sync.h" , "support/atm_windows.h" , "impl/codegen/atm.h" , "impl/codegen/atm_gcc_atomic.h" , "impl/codegen/atm_gcc_sync.h" , "impl/codegen/atm_windows.h" ] , "stage": ["grpc"] } , "slice_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["slice.h"] , "stage": ["grpc"] } , "census_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["census.h"] , "stage": ["grpc"] } , "channel_arg_names_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["impl/channel_arg_names.h"] , "stage": ["grpc"] } , "grpc_core_credentials_header": { "type": ["@", "rules", "data", "staged"] , "srcs": ["credentials.h"] , "stage": ["grpc"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpcpp/000077500000000000000000000000001516554100600246225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/include/grpcpp/TARGETS.grpc000066400000000000000000000130301516554100600266050ustar00rootroot00000000000000{ "grpcpp_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "alarm.h" , "channel.h" , "client_context.h" , "completion_queue.h" , "create_channel.h" , "create_channel_posix.h" , "ext/health_check_service_server_builder_option.h" , "generic/async_generic_service.h" , "generic/callback_generic_service.h" , "generic/generic_stub.h" , "generic/generic_stub_callback.h" , "grpcpp.h" , "health_check_service_interface.h" , "impl/call_hook.h" , "impl/call_op_set_interface.h" , "impl/call_op_set.h" , "impl/call.h" , "impl/channel_argument_option.h" , "impl/channel_interface.h" , "impl/client_unary_call.h" , "impl/completion_queue_tag.h" , "impl/create_auth_context.h" , "impl/delegating_channel.h" , "impl/grpc_library.h" , "impl/intercepted_channel.h" , "impl/interceptor_common.h" , "impl/metadata_map.h" , "impl/method_handler_impl.h" , "impl/rpc_method.h" , "impl/rpc_service_method.h" , "impl/serialization_traits.h" , "impl/server_builder_option.h" , "impl/server_builder_plugin.h" , "impl/server_callback_handlers.h" , "impl/server_initializer.h" , "impl/service_type.h" , "impl/status.h" , "impl/sync.h" , "passive_listener.h" , "resource_quota.h" , "security/audit_logging.h" , "security/tls_crl_provider.h" , "security/auth_context.h" , "security/auth_metadata_processor.h" , "security/credentials.h" , "security/server_credentials.h" , "security/tls_certificate_provider.h" , "security/authorization_policy_provider.h" , "security/tls_certificate_verifier.h" , "security/tls_credentials_options.h" , "server.h" , "server_builder.h" , "server_context.h" , "server_interface.h" , "server_posix.h" , "version_info.h" , "support/async_stream.h" , "support/async_unary_call.h" , "support/byte_buffer.h" , "support/callback_common.h" , "support/channel_arguments.h" , "support/client_callback.h" , "support/client_interceptor.h" , "support/config.h" , "support/interceptor.h" , "support/message_allocator.h" , "support/method_handler.h" , "support/proto_buffer_reader.h" , "support/proto_buffer_writer.h" , "support/server_callback.h" , "support/server_interceptor.h" , "support/slice.h" , "support/status.h" , "support/status_code_enum.h" , "support/string_ref.h" , "support/stub_options.h" , "support/sync_stream.h" , "support/time.h" , "support/validate_service_config.h" , "impl/codegen/async_generic_service.h" , "impl/codegen/async_stream.h" , "impl/codegen/async_unary_call.h" , "impl/codegen/byte_buffer.h" , "impl/codegen/call_hook.h" , "impl/codegen/call_op_set_interface.h" , "impl/codegen/call_op_set.h" , "impl/codegen/call.h" , "impl/codegen/callback_common.h" , "impl/codegen/channel_interface.h" , "impl/codegen/client_callback.h" , "impl/codegen/client_context.h" , "impl/codegen/client_interceptor.h" , "impl/codegen/client_unary_call.h" , "impl/codegen/completion_queue_tag.h" , "impl/codegen/completion_queue.h" , "impl/codegen/config.h" , "impl/codegen/create_auth_context.h" , "impl/codegen/delegating_channel.h" , "impl/codegen/intercepted_channel.h" , "impl/codegen/interceptor_common.h" , "impl/codegen/interceptor.h" , "impl/codegen/message_allocator.h" , "impl/codegen/metadata_map.h" , "impl/codegen/method_handler_impl.h" , "impl/codegen/method_handler.h" , "impl/codegen/rpc_method.h" , "impl/codegen/rpc_service_method.h" , "impl/codegen/security/auth_context.h" , "impl/codegen/serialization_traits.h" , "impl/codegen/server_callback_handlers.h" , "impl/codegen/server_callback.h" , "impl/codegen/server_context.h" , "impl/codegen/server_interceptor.h" , "impl/codegen/server_interface.h" , "impl/codegen/service_type.h" , "impl/codegen/slice.h" , "impl/codegen/status_code_enum.h" , "impl/codegen/status.h" , "impl/codegen/string_ref.h" , "impl/codegen/stub_options.h" , "impl/codegen/sync_stream.h" , "impl/codegen/time.h" , "impl/codegen/sync.h" ] , "stage": ["grpcpp"] } , "grpcpp_config_proto_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["impl/codegen/config_protobuf.h"] , "stage": ["grpcpp"] } , "grpcpp_codegen_proto_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "impl/codegen/proto_buffer_reader.h" , "impl/codegen/proto_buffer_writer.h" , "impl/codegen/proto_utils.h" , "impl/generic_serialize.h" , "impl/proto_utils.h" ] , "stage": ["grpcpp"] } , "grpc++_binder_grpcpp_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "security/binder_security_policy.h" , "create_channel_binder.h" , "security/binder_credentials.h" ] , "stage": ["grpcpp"] } , "grpcpp_call_metric_recorder_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["ext/call_metric_recorder.h"] , "stage": ["grpcpp"] } , "grpcpp_backend_metric_recorder_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["ext/server_metric_recorder.h"] , "stage": ["grpcpp"] } , "global_callback_hook_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["support/global_callback_hook.h"] , "stage": ["grpcpp"] } , "grpcpp_xds_server_builder_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["xds_server_builder.h"] , "stage": ["grpcpp"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/lib/000077500000000000000000000000001516554100600224525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/lib/TARGETS.curl000066400000000000000000001266351516554100600244670ustar00rootroot00000000000000{ "curl_config.h": { "type": "configure" , "arguments_config": [ "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "CURL_HIDDEN_SYMBOLS" , "USE_ZLIB" , "ENABLE_ARES" , "ENABLE_THREADED_RESOLVER" , "CURL_DISABLE_DICT" , "CURL_DISABLE_FILE" , "CURL_DISABLE_FORM_API" , "CURL_DISABLE_FTP" , "CURL_DISABLE_GOPHER" , "CURL_DISABLE_IMAP" , "CURL_DISABLE_LDAP" , "CURL_DISABLE_LDAPS" , "CURL_DISABLE_MQTT" , "CURL_DISABLE_POP3" , "CURL_DISABLE_RTSP" , "CURL_DISABLE_SMB" , "CURL_DISABLE_SMTP" , "CURL_DISABLE_TELNET" , "CURL_DISABLE_TFTP" , "HTTP_ONLY" , "CURL_DISABLE_ALTSVC" , "CURL_DISABLE_SRP" , "CURL_DISABLE_COOKIES" , "CURL_DISABLE_BASIC_AUTH" , "CURL_DISABLE_BEARER_AUTH" , "CURL_DISABLE_DIGEST_AUTH" , "CURL_DISABLE_KERBEROS_AUTH" , "CURL_DISABLE_NEGOTIATE_AUTH" , "CURL_DISABLE_AWS" , "CURL_DISABLE_NTLM" , "CURL_DISABLE_DOH" , "CURL_DISABLE_GETOPTIONS" , "CURL_DISABLE_HEADERS_API" , "CURL_DISABLE_BINDLOCAL" , "CURL_DISABLE_HSTS" , "CURL_DISABLE_MIME" , "CURL_DISABLE_NETRC" , "CURL_DISABLE_PARSEDATE" , "CURL_DISABLE_PROGRESS_METER" , "CURL_DISABLE_SHUFFLE_DNS" , "CURL_DISABLE_SOCKETPAIR" , "ENABLE_IPV6" , "CURL_ENABLE_SSL" , "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , "USE_NGHTTP2" , "USE_NGTCP2" , "USE_QUICHE" , "USE_MSH3" , "USE_LIBIDN2" , "USE_BROTLI" , "CURL_ZSTD" , "CURL_USE_LIBPSL" , "CURL_USE_LIBSSH2" , "CURL_USE_LIBSSH" , "CURL_USE_GSSAPI" , "ENABLE_UNIX_SOCKETS" , "CURL_CA_BUNDLE" , "CURL_CA_PATH" , "CURL_CA_FALLBACK" , "USE_GNU_STRERROR_R" , "HAVE_BORINGSSL" , "HAVE_AWSLC" , "HAVE_SSL_SET0_WBIO" , "HAVE_OPENSSL_SRP" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" ] , "target": "config_header" , "config": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } ] , [ "sys_hdrs" , { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , "then": ["windows.h", "ws2tcpip.h", "winsock2.h"] } , [ "sys/filio.h" , "sys/wait.h" , "sys/ioctl.h" , "sys/param.h" , "sys/poll.h" , "sys/resource.h" , "sys/select.h" , "sys/socket.h" , "sys/sockio.h" , "sys/stat.h" , "sys/time.h" , "sys/types.h" , "sys/un.h" , "sys/utime.h" , "sys/xattr.h" , "arpa/inet.h" , "assert.h" , "errno.h" , "fcntl.h" , "fnmatch.h" , "idn2.h" , "ifaddrs.h" , "io.h" , "libgen.h" , "locale.h" , "net/if.h" , "netdb.h" , "netinet/in.h" , "netinet/tcp.h" , "netinet/udp.h" , "linux/tcp.h" , "poll.h" , "pwd.h" , "setjmp.h" , "sched.h" , "signal.h" , "signal.h" , "ssl.h" , "signal.h" , "ssl.h" , "stdatomic.h" , "stdbool.h" , "stdint.h" , "stdlib.h" , "string.h" , "strings.h" , "stropts.h" , "termio.h" , "termios.h" , "time.h" , "unistd.h" , "utime.h" , "process.h" ] , { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_GSSAPI"} , "then": [ "gssapi/gssapi.h" , "gssapi/gssapi_generic.h" , "gssapi/gssapi_krb5.h" ] } ] } ] , [ "defines1" , { "type": "++" , "$1": [ [ ["BUILDING_LIBCURL", 1] , ["HAVE_BOOL_T", 1] , ["HAVE_RECV", 1] , ["HAVE_SEND", 1] , ["HAVE_STRUCT_TIMEVAL", 1] , ["HAVE_WRITABLE_ARGV", 1] , ["STDC_HEADERS", 1] , ["HAVE_VARIADIC_MACROS_C99", 1] , ["HAVE_VARIADIC_MACROS_GCC", 1] , ["HAVE_RAND_EGD", {"type": "var", "name": "HAVE_RAND_EGD"}] , ["HAVE_BORINGSSL", {"type": "var", "name": "HAVE_BORINGSSL"}] , ["HAVE_AWSLC", {"type": "var", "name": "HAVE_AWSLC"}] , ["HAVE_LIBZ", {"type": "var", "name": "USE_ZLIB"}] ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADED_RESOLVER"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["windows", [["USE_THREADS_WIN32", 1]]]] , "default": [["USE_THREADS_POSIX", 1], ["HAVE_PTHREAD_H", 1]] } } , { "type": "if" , "cond": {"type": "var", "name": "HTTP_ONLY"} , "then": [ ["CURL_DISABLE_DICT", 1] , ["CURL_DISABLE_FILE", 1] , ["CURL_DISABLE_FTP", 1] , ["CURL_DISABLE_GOPHER", 1] , ["CURL_DISABLE_IMAP", 1] , ["CURL_DISABLE_LDAP", 1] , ["CURL_DISABLE_LDAPS", 1] , ["CURL_DISABLE_MQTT", 1] , ["CURL_DISABLE_POP3", 1] , ["CURL_DISABLE_RTSP", 1] , ["CURL_DISABLE_SMB", 1] , ["CURL_DISABLE_SMTP", 1] , ["CURL_DISABLE_TELNET", 1] , ["CURL_DISABLE_TFTP", 1] ] , "else": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_DICT"} , "then": [["CURL_DISABLE_DICT", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_FILE"} , "then": [["CURL_DISABLE_FILE", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_FTP"} , "then": [["CURL_DISABLE_FTP", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_GOPHER"} , "then": [["CURL_DISABLE_GOPHER", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_IMAP"} , "then": [["CURL_DISABLE_IMAP", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_LDAP"} , "then": [["CURL_DISABLE_LDAP", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_LDAPS"} , "then": [["CURL_DISABLE_LDAPS", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_MQTT"} , "then": [["CURL_DISABLE_MQTT", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_POP3"} , "then": [["CURL_DISABLE_POP3", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_RTSP"} , "then": [["CURL_DISABLE_RTSP", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_SMB"} , "then": [["CURL_DISABLE_SMB", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_SMTP"} , "then": [["CURL_DISABLE_SMTP", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_TELNET"} , "then": [["CURL_DISABLE_TELNET", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_TFTP"} , "then": [["CURL_DISABLE_TFTP", 1]] } ] } } , [ ["USE_ARES", {"type": "var", "name": "ENABLE_ARES"}] , [ "CURL_DISABLE_ALTSVC" , {"type": "var", "name": "CURL_DISABLE_ALTSVC"} ] , [ "CURL_DISABLE_COOKIES" , {"type": "var", "name": "CURL_DISABLE_COOKIES"} ] , [ "CURL_DISABLE_NTLM" , {"type": "var", "name": "CURL_DISABLE_NTLM"} ] , [ "CURL_DISABLE_DOH" , {"type": "var", "name": "CURL_DISABLE_DOH"} ] , [ "CURL_DISABLE_GETOPTIONS" , {"type": "var", "name": "CURL_DISABLE_GETOPTIONS"} ] , [ "CURL_DISABLE_HSTS" , {"type": "var", "name": "CURL_DISABLE_HSTS"} ] , [ "CURL_DISABLE_MIME" , {"type": "var", "name": "CURL_DISABLE_MIME"} ] , [ "CURL_DISABLE_NETRC" , {"type": "var", "name": "CURL_DISABLE_NETRC"} ] , [ "CURL_DISABLE_PARSEDATE" , {"type": "var", "name": "CURL_DISABLE_PARSEDATE"} ] , [ "CURL_DISABLE_PROGRESS_METER" , {"type": "var", "name": "CURL_DISABLE_PROGRESS_METER"} ] , [ "CURL_DISABLE_SHUFFLE_DNS" , {"type": "var", "name": "CURL_DISABLE_SHUFFLE_DNS"} ] , [ "CURL_DISABLE_SOCKETPAIR" , {"type": "var", "name": "CURL_DISABLE_SOCKETPAIR"} ] , ["ENABLE_IPV6", {"type": "var", "name": "ENABLE_IPV6"}] , [ "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" , { "type": "var" , "name": "CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG" } ] , ["USE_NGHTTP2", {"type": "var", "name": "USE_NGHTTP2"}] , ["HAVE_LIBIDN2", {"type": "var", "name": "USE_LIBIDN2"}] , ["HAVE_BROTLI", {"type": "var", "name": "USE_BROTLI"}] , ["HAVE_ZSTD", {"type": "var", "name": "CURL_ZSTD"}] , [ "CURL_CA_FALLBACK" , {"type": "var", "name": "CURL_CA_FALLBACK"} ] ] , { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": [["USE_OPENSSL", 1], ["SSL_ENABLED", 1]] } , { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": [["USE_NGTCP2", 1], ["USE_NGHTTP3", 1]] , "else": { "type": "fail" , "msg": "libcurl: ngtcp2 requires OpenSSL/BoringSSL." } } } , { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "fail" , "msg": "libcurl: only one HTTP/3 backend can be selected, but both quiche and ngtcp2 given." } , "else": [ [ "HAVE_SSL_CTX_SET_QUIC_METHOD" , {"type": "var", "name": "HAVE_SSL_CTX_SET_QUIC_METHOD"} ] , ["USE_QUICHE", 1] , [ "HAVE_QUICHE_CONN_SET_QLOG_FD" , {"type": "var", "name": "HAVE_QUICHE_CONN_SET_QLOG_FD"} ] ] } } , { "type": "if" , "cond": {"type": "var", "name": "USE_MSH3"} , "then": { "type": "if" , "cond": { "type": "or" , "$1": [ {"type": "var", "name": "USE_NGTCP2"} , {"type": "var", "name": "USE_QUICHE"} ] } , "then": { "type": "fail" , "msg": "libcurl: only one HTTP/3 backend can be selected, but both msh3 and ngtcp2 or quiche given." } , "else": [["USE_MSH3", 1]] } } , { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_SRP"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "HAVE_OPENSSL_SRP"} , "then": [["USE_TLS_SRP", 1]] } } , [ ["USE_LIBPSL", {"type": "var", "name": "CURL_USE_LIBPSL"}] , ["USE_LIBSSH2", {"type": "var", "name": "CURL_USE_LIBSSH2"}] ] , { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH"} , "then": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH2"} , "then": [] , "else": [["USE_LIBSSH", 1]] } } , { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_GSSAPI"} , "then": [["HAVE_GSSAPI", 1], ["HAVE_GSSHEIMDAL", 1]] } , [ [ "USE_WIN32_LARGE_FILES" , { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } ] , [ "HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID" , { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_IPV6"} , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , "then": false , "else": true } ] } ] ] , { "type": "if" , "cond": {"type": "var", "name": "USE_GNU_STRERROR_R"} , "then": [["HAVE_GLIBC_STRERROR_R", 1]] , "else": [["HAVE_POSIX_STRERROR_R", 1]] } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "clang" } , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ ["mac", [["HAVE_BUILTIN_AVAILABLE", 1]]] , ["android", [["HAVE_BUILTIN_AVAILABLE", 1]]] ] } } , [["HAVE_SA_FAMILY_T", 1]] ] } ] , [ "have_cfile" , { "type": "++" , "$1": [ [ ["HAVE_SYS_FILIO_H", "sys/filio.h"] , ["HAVE_SYS_WAIT_H", "sys/wait.h"] , ["HAVE_SYS_IOCTL_H", "sys/ioctl.h"] , ["HAVE_SYS_PARAM_H", "sys/param.h"] , ["HAVE_SYS_POLL_H", "sys/poll.h"] , ["HAVE_SYS_RESOURCE_H", "sys/resource.h"] , ["HAVE_SYS_SELECT_H", "sys/select.h"] , ["HAVE_SYS_SOCKET_H", "sys/socket.h"] , ["HAVE_SYS_SOCKIO_H", "sys/sockio.h"] , ["HAVE_SYS_STAT_H", "sys/stat.h"] , ["HAVE_SYS_TIME_H", "sys/time.h"] , ["HAVE_SYS_TYPES_H", "sys/types.h"] , ["HAVE_SYS_UN_H", "sys/un.h"] , ["HAVE_SYS_UTIME_H", "sys/utime.h"] , ["HAVE_SYS_XATTR_H", "sys/xattr.h"] , ["HAVE_ARPA_INET_H", "arpa/inet.h"] , ["HAVE_FCNTL_H", "fcntl.h"] , ["HAVE_IDN2_H", "idn2.h"] , ["HAVE_IFADDRS_H", "ifaddrs.h"] , ["HAVE_IO_H", "io.h"] , ["HAVE_LIBGEN_H", "libgen.h"] , ["HAVE_LOCALE_H", "locale.h"] , ["HAVE_NET_IF_H", "net/if.h"] , ["HAVE_NETDB_H", "netdb.h"] , ["HAVE_NETINET_IN_H", "netinet/in.h"] , ["HAVE_NETINET_TCP_H", "netinet/tcp.h"] , ["HAVE_NETINET_UDP_H", "netinet/udp.h"] , ["HAVE_LINUX_TCP_H", "linux/tcp.h"] , ["HAVE_POLL_H", "poll.h"] , ["HAVE_PWD_H", "pwd.h"] , ["HAVE_STDATOMIC_H", "stdatomic.h"] , ["HAVE_STDBOOL_H", "stdbool.h"] , ["HAVE_STRINGS_H", "strings.h"] , ["HAVE_STROPTS_H", "stropts.h"] , ["HAVE_TERMIO_H", "termio.h"] , ["HAVE_TERMIOS_H", "termios.h"] , ["HAVE_UNISTD_H", "unistd.h"] , ["HAVE_UTIME_H", "utime.h"] , ["HAVE_ATOMIC", "stdatomic.h"] ] , { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_GSSAPI"} , "then": [ ["HAVE_GSSAPI_GSSAPI_GENERIC_H", "gssapi/gssapi_generic.h"] , ["HAVE_GSSAPI_GSSAPI_H", "gssapi/gssapi.h"] , ["HAVE_GSSAPI_GSSAPI_KRB5_H", "gssapi/gssapi_krb5.h"] ] } ] } ] , [ "have_csymbol" , { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "HAVE_RAND_EGD"} , "then": [] , "else": [ [ "HAVE_RAND_EGD" , ["RAND_egd", {"type": "var", "name": "sys_hdrs"}] ] ] } } , { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "HAVE_BORINGSSL"} , "then": [] , "else": [ [ "HAVE_BORINGSSL" , ["OPENSSL_IS_BORINGSSL", ["openssl/base.h"]] ] ] } } , { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "HAVE_AWSLC"} , "then": [] , "else": [["HAVE_AWSLC", ["OPENSSL_IS_AWSLC", ["openssl/base.h"]]]] } } , { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "fail" , "msg": "libcurl: only one HTTP/3 backend can be selected, but both quiche and ngtcp2 given." } , "else": { "type": "if" , "cond": {"type": "var", "name": "HAVE_SSL_CTX_SET_QUIC_METHOD"} , "then": [] , "else": [ [ "HAVE_SSL_CTX_SET_QUIC_METHOD" , ["SSL_CTX_set_quic_method", ["openssl/ssl.h"]] ] ] } } } , { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "fail" , "msg": "libcurl: only one HTTP/3 backend can be selected, but both quiche and ngtcp2 given." } , "else": { "type": "if" , "cond": {"type": "var", "name": "HAVE_QUICHE_CONN_SET_QLOG_FD"} , "then": [] , "else": [ [ "HAVE_QUICHE_CONN_SET_QLOG_FD" , ["quiche_conn_set_qlog_fd", ["quiche.h"]] ] ] } } } , { "type": "if" , "cond": {"type": "var", "name": "HAVE_SSL_SET0_WBIO"} , "then": [] , "else": [["HAVE_SSL_SET0_WBIO", ["SSL_set0_wbio", ["openssl/ssl.h"]]]] } , { "type": "if" , "cond": {"type": "var", "name": "HAVE_OPENSSL_SRP"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "CURL_DISABLE_SRP"} , "then": [] , "else": [ [ "HAVE_OPENSSL_SRP" , ["SSL_CTX_set_srp_username", ["openssl/ssl.h"]] ] ] } } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , "then": [ [ "USE_WIN32_CRYPTO" , ["CryptAcquireContext", ["windows.h", "wincrypt.h"]] ] ] } , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_UNIX_SOCKETS"} , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , "then": false , "else": true } ] } , "then": [ [ "USE_UNIX_SOCKETS" , [ "((struct sockaddr_un*)NULL)->sun_path" , {"type": "var", "name": "sys_hdrs"} ] ] ] } , [ [ "HAVE_FNMATCH" , ["fnmatch", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_BASENAME" , ["basename", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SOCKET" , ["socket", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SCHED_YIELD" , ["sched_yield", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SELECT" , ["select", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRDUP" , ["strdup", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRTOK_R" , ["strtok_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRCASECMP" , ["strcasecmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRICMP" , ["stricmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRCMPI" , ["strccmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MEMRCHR" , ["memrchr", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_ALARM", ["alarm", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_ARC4RANDOM" , ["arc4random", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_FCNTL", ["fcntl", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_GETPPID" , ["getppid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_UTIMES" , ["utimes", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETTIMEOFDAY" , ["gettimeofday", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_CLOSESOCKET" , ["closesocket", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SIGSETJMP" , ["sigsetjmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPASS_R" , ["getpass_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPWUID" , ["getpwuid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPWUID_R" , ["getpwuid_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETEUID" , ["geteuid", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_UTIME", ["utime", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_GMTIME_R" , ["gmtime_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETHOSTBYNAME_R" , ["gethostbyname_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SIGNAL" , ["signal", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRTOLL" , ["strtoll", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRERROR_R" , ["strerror_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SIGINTERRUPT" , ["siginterrupt", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETADDRINFO" , ["getaddrinfo", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETIFADDRS" , ["getifaddrs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FREEADDRINFO" , ["freeaddrinfo", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_PIPE", ["pipe", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_FTRUNCATE" , ["ftruncate", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSEEKO" , ["fseeko", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_DECL_FSEEKO" , ["fseeko", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE__FSEEKI64" , ["_fseeki64", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPEERNAME" , ["getpeername", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETSOCKNAME" , ["getsockname", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_IF_NAMETOINDEX" , ["if_nametoindex", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETRLIMIT" , ["getrlimit", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETLOCALE" , ["setlocale", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETMODE" , ["setmode", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETRLIMIT" , ["setrlimit", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MACH_ABSOLUTE_TIME" , ["mach_absolute_time", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_INET_NTOP" , ["inet_ntop", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_INET_PTON" , ["inet_pton", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSETXATTR" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_SIGACTION", ["sigaction", ["signal.h"]]] , [ "HAVE_FCNTL_O_NONBLOCK" , ["O_NONBLOCK", ["sys/types.h", "unistd.h", "fcntl.h"]] ] , [ "HAVE_FNMATCH" , ["fnmatch", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SENDMSG" , ["sendmsg", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SOCKETPAIR" , ["socketpair", {"type": "var", "name": "sys_hdrs"}] ] ] , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "msvc" } , "then": [["HAVE_SNPRINTF", ["snprintf", ["stdio.h"]]]] } , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "linux" , [ [ "HAVE_FSETXATTR_5" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] ] ] , [ "windows" , [ [ "HAVE_FSETXATTR_5" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] ] ] , [ "mac" , [ [ "HAVE_FSETXATTR_6" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] ] ] ] } , [ [ "HAVE_IOCTLSOCKET" , ["ioctlsocket", ["windows.h", "winsock2.h"]] ] , [ "HAVE_IOCTLSOCKET_CAMEL" , ["IoctlSocket", ["windows.h", "winsock2.h"]] ] , [ "HAVE_IOCTLSOCKET_FIONBIO" , ["FIONBIO", ["windows.h", "winsock2.h"]] ] , [ "HAVE_IOCTLSOCKET_CAMEL_FIONBIO" , ["FIONBIO", ["windows.h", "winsock2.h"]] ] , [ "HAVE_IOCTL_FIONBIO" , [ "FIONBIO" , [ "sys/types.h" , "unistd.h" , "sys/socket.h" , "sys/ioctl.h" , "stropts.h" ] ] ] , [ "HAVE_IOCTL_SIOCGIFADDR" , [ "SIOCGIFADDR" , [ "sys/types.h" , "unistd.h" , "sys/socket.h" , "sys/ioctl.h" , "stropts.h" , "net/if.h" ] ] ] , [ "HAVE_SETSOCKOPT_SO_NONBLOCK" , [ "SO_NONBLOCK" , ["windows.h", "winsock2.h", "sys/types.h", "sys/socket.h"] ] ] , [ "HAVE_CLOCK_GETTIME_MONOTONIC" , ["CLOCK_MONOTONIC", ["time.h"]] ] , [ "HAVE_WIN32_WINNT" , ["_WIN32_WINNT", {"type": "var", "name": "sys_hdrs"}] ] ] ] } ] , ["have_ctype", [["HAVE_LONGLONG", "long long"]]] , ["have_ctype", [["HAVE_SUSECONDS_T", "suseconds_t"]]] , [ "size_ctype" , [ ["SIZEOF_SIZE_T", ["size_t", [1, 2, 4, 8, 16]]] , ["SIZEOF_SSIZE_T", ["ssize_t", [1, 2, 4, 8, 16]]] , ["SIZEOF_LONG_LONG", ["long long", [1, 2, 4, 8, 16]]] , ["SIZEOF_LONG", ["long", [1, 2, 4, 8, 16]]] , ["SIZEOF_INT", ["int", [1, 2, 4, 8, 16]]] , ["SIZEOF_TIME_T", ["time_t", [1, 2, 4, 8, 16]]] , ["SIZEOF_SUSECONDS_T", ["suseconds_t", [1, 2, 4, 8, 16]]] , ["SIZEOF_OFF_T", ["off_t", [1, 2, 4, 8, 16]]] ] ] , [ "defines" , { "type": "++" , "$1": [ [ [ "OS" , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "case*" , "expr": {"type": "var", "name": "TARGET_ARCH"} , "case": [ ["x86", "\"i386-pc-win32\""] , ["x86_64", "\"x86_64-pc-win32\""] , ["arm", "\"thumbv7a-pc-win32\""] , ["arm64", "\"aarch64-pc-win32\""] ] , "default": "\"unknown-pc-win32\"" } ] , [ "linux" , { "type": "join" , "separator": "" , "$1": { "type": "++" , "$1": [ { "type": "case*" , "expr": {"type": "var", "name": "TARGET_ARCH"} , "case": [ ["x86", ["\"i386"]] , ["x86_64", ["\"x86_64"]] , ["arm", ["\"thumbv7a"]] , ["arm64", ["\"aarch64"]] ] , "default": ["\"unknown"] } , ["-pc-linux"] , { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [["gnu", ["-gnu\""]], ["clang", ["-clang\""]]] , "default": ["\""] } ] } } ] , ["mac", "\"mac\""] ] } ] ] , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "linux" , [ ["RECV_TYPE_ARG1", "int"] , ["RECV_TYPE_ARG2", "void *"] , ["RECV_TYPE_ARG3", "size_t"] , ["RECV_TYPE_ARG4", "int"] , ["RECV_TYPE_RETV", "ssize_t"] , ["SELECT_QUAL_ARG5", " "] , ["SELECT_TYPE_ARG1", "int"] , ["SELECT_TYPE_ARG234", "fd_set *"] , ["SELECT_TYPE_ARG5", "struct timeval *"] , ["SELECT_TYPE_RETV", "int"] , ["SEND_QUAL_ARG2", "const"] , ["SEND_TYPE_ARG1", "int"] , ["SEND_TYPE_ARG2", "void *"] , ["SEND_TYPE_ARG3", "size_t"] , ["SEND_TYPE_ARG4", "int"] , ["SEND_TYPE_RETV", "ssize_t"] ] ] , [ "windows" , [ ["RECV_TYPE_ARG1", "SOCKET"] , ["RECV_TYPE_ARG2", "char *"] , ["RECV_TYPE_ARG3", "int"] , ["RECV_TYPE_ARG4", "int"] , ["RECV_TYPE_RETV", "int"] , ["SEND_QUAL_ARG2", "const"] , ["SEND_TYPE_ARG1", "SOCKET"] , ["SEND_TYPE_ARG2", "char *"] , ["SEND_TYPE_ARG3", "int"] , ["SEND_TYPE_ARG4", "int"] , ["SEND_TYPE_RETV", "int"] ] ] ] } , { "type": "case*" , "expr": {"type": "var", "name": "CURL_CA_BUNDLE"} , "case": [ ["auto", []] , ["none", []] , [ "" , { "type": "fail" , "msg": "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path." } ] ] , "default": [["CURL_CA_BUNDLE", {"type": "var", "name": "CURL_CA_BUNDLE"}]] } , { "type": "case*" , "expr": {"type": "var", "name": "CURL_CA_PATH"} , "case": [ ["auto", []] , ["none", []] , [ "" , { "type": "fail" , "msg": "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or file path." } ] ] , "default": [["CURL_CA_PATH", {"type": "var", "name": "CURL_CA_PATH"}]] } , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_UNIX_SOCKETS"} , { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } ] } , "then": [["USE_UNIX_SOCKETS", " "]] } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "msvc" } , "then": [["CURL_EXTERN_SYMBOL", " "]] , "else": { "type": "if" , "cond": {"type": "var", "name": "CURL_HIDDEN_SYMBOLS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [ [ "clang" , [ [ "CURL_EXTERN_SYMBOL" , "__attribute__ ((__visibility__ (\"default\")))" ] ] ] , [ "gnu" , [ [ "CURL_EXTERN_SYMBOL" , "__attribute__ ((__visibility__ (\"default\")))" ] ] ] , ["sunpro", [["CURL_EXTERN_SYMBOL", "__global"]]] , [ "intel" , [ [ "CURL_EXTERN_SYMBOL" , "__attribute__ ((__visibility__ (\"default\")))" ] ] ] ] , "default": [["CURL_EXTERN_SYMBOL", " "]] } , "else": [["CURL_EXTERN_SYMBOL", " "]] } } , [["SIZEOF_CURL_OFF_T", "8"]] , [["SIZEOF_CURL_SOCKET_T", "4"]] , [["SIZEOF_SA_FAMILY_T", "2"]] ] } ] ] , "body": { "type": "env" , "vars": [ "defines1" , "have_cfile" , "have_csymbol" , "have_ctype" , "size_ctype" , "defines" ] } } } , "config_header": { "type": ["@", "rules", "CC/auto", "config"] , "name": ["curl_config.h"] , "guard": ["INCLUDE_curl_config_h__"] , "deps": [ ["src", "curl_ares"] , ["src", "curl_async_dns"] , ["src", "curl_brotli"] , ["src", "curl_crypt_win32"] , ["src", "curl_gssapi"] , ["src", "curl_hidden_symbols"] , ["src", "curl_idn2"] , ["src", "curl_ipv6"] , ["src", "curl_msh3"] , ["src", "curl_nghttp2"] , ["src", "curl_ngtcp2"] , ["src", "curl_psl"] , ["src", "curl_quiche"] , ["src", "curl_ssh"] , ["src", "curl_ssh2"] , ["src", "curl_ssl"] , ["src", "curl_zlib"] , ["src", "curl_zstd"] ] } , "libcurl_csources": {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.c"]]} , "libcurl_hheaders": {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.h"]]} , "curl_vauth_cfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "vauth/cleartext.c" , "vauth/cram.c" , "vauth/digest.c" , "vauth/digest_sspi.c" , "vauth/gsasl.c" , "vauth/krb5_gssapi.c" , "vauth/krb5_sspi.c" , "vauth/ntlm.c" , "vauth/ntlm_sspi.c" , "vauth/oauth2.c" , "vauth/spnego_gssapi.c" , "vauth/spnego_sspi.c" , "vauth/vauth.c" ] } , "curl_vauth_hfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": ["vauth/digest.h", "vauth/ntlm.h", "vauth/vauth.h"] } , "curl_vtls_cfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "vtls/bearssl.c" , "vtls/gtls.c" , "vtls/hostcheck.c" , "vtls/keylog.c" , "vtls/mbedtls.c" , "vtls/mbedtls_threadlock.c" , "vtls/openssl.c" , "vtls/rustls.c" , "vtls/schannel.c" , "vtls/schannel_verify.c" , "vtls/sectransp.c" , "vtls/vtls.c" , "vtls/wolfssl.c" , "vtls/x509asn1.c" ] } , "curl_vtls_hfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "vtls/bearssl.h" , "vtls/gtls.h" , "vtls/hostcheck.h" , "vtls/keylog.h" , "vtls/mbedtls.h" , "vtls/mbedtls_threadlock.h" , "vtls/openssl.h" , "vtls/rustls.h" , "vtls/schannel.h" , "vtls/schannel_int.h" , "vtls/sectransp.h" , "vtls/vtls.h" , "vtls/vtls_int.h" , "vtls/wolfssl.h" , "vtls/x509asn1.h" ] } , "curl_vquic_cfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "vquic/curl_msh3.c" , "vquic/curl_ngtcp2.c" , "vquic/curl_osslq.c" , "vquic/curl_quiche.c" , "vquic/vquic.c" , "vquic/vquic-tls.c" ] } , "curl_vquic_hfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "vquic/curl_msh3.h" , "vquic/curl_ngtcp2.h" , "vquic/curl_osslq.h" , "vquic/curl_quiche.h" , "vquic/vquic.h" , "vquic/vquic_int.h" , "vquic/vquic-tls.h" ] } , "curl_vssh_cfiles": { "type": ["@", "rules", "data", "staged"] , "srcs": ["vssh/libssh.c", "vssh/libssh2.c", "vssh/wolfssh.c"] } , "curl_vssh_hfiles": {"type": ["@", "rules", "data", "staged"], "srcs": ["vssh/ssh.h"]} , "libcurl": { "type": ["@", "rules", "CC", "library"] , "name": ["libcurl"] , "pure C": ["yes"] , "srcs": [ "libcurl_csources" , "curl_vauth_cfiles" , "curl_vquic_cfiles" , "curl_vssh_cfiles" , "curl_vtls_cfiles" ] , "hdrs": [["include/curl", "curl_public_headers"]] , "private-hdrs": [ "libcurl_hheaders" , "curl_vauth_hfiles" , "curl_vquic_hfiles" , "curl_vssh_hfiles" , "curl_vtls_hfiles" , "curl_config.h" , ["include/curl", "curl_public_headers"] ] , "deps": [ ["src", "curl_ares"] , ["src", "curl_async_dns"] , ["src", "curl_brotli"] , ["src", "curl_crypt_win32"] , ["src", "curl_gssapi"] , ["src", "curl_hidden_symbols"] , ["src", "curl_idn2"] , ["src", "curl_ipv6"] , ["src", "curl_msh3"] , ["src", "curl_nghttp2"] , ["src", "curl_ngtcp2"] , ["src", "curl_psl"] , ["src", "curl_quiche"] , ["src", "curl_ssh"] , ["src", "curl_ssh2"] , ["src", "curl_ssl"] , ["src", "curl_zlib"] , ["src", "curl_zstd"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/libarchive/000077500000000000000000000000001516554100600240145ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/libarchive/TARGETS.archive000066400000000000000000001761561516554100600265100ustar00rootroot00000000000000{ "config.h": { "type": "configure" , "arguments_config": [ "OS" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "ENABLE_MBEDTLS" , "ENABLE_NETTLE" , "ENABLE_OPENSSL" , "ENABLE_LIBB2" , "ENABLE_LZ4" , "ENABLE_LZO" , "ENABLE_LZMA" , "ENABLE_ZSTD" , "ENABLE_ZLIB" , "ENABLE_BZip2" , "ENABLE_LIBXML2" , "ENABLE_EXPAT" , "ENABLE_PCREPOSIX" , "ENABLE_PCRE2POSIX" , "ENABLE_LIBGCC" , "ENABLE_CNG" , "ENABLE_XATTR" , "ENABLE_ACL" , "ENABLE_ICONV" , "ENABLE_LIBMD" , "ENABLE_PCRE" , "ENABLE_PCRE2" , "ENABLE_REGEX" , "XATTR_PROVIDER" , "ENABLE_RICHACL" , "USE_NFS4" ] , "target": "config_header" , "config": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } ] , [ "sys_hdrs" , [ "zlib.h" , "bzlib.h" , "lzma.h" , "lzo/lzoconf.h" , "lzo/lzo1x.h" , "blake.h" , "lz4.h" , "lz4hc.h" , "zstd.h" , "dirent.h" , "sys/ndir.h" , "ndir.h" , "sys/dir.h" , "sys/types.h" , "acl/libacl.h" , "attr/xattr.h" , "ctype.h" , "copyfile.h" , "direct.h" , "dlfcn.h" , "errno.h" , "ext2fs/ext2_fs.h" , "fcntl.h" , "fnmatch.h" , "grp.h" , "inttypes.h" , "io.h" , "langinfo.h" , "limits.h" , "linux/types.h" , "linux/fiemap.h" , "linux/fs.h" , "linux/magic.h" , "locale.h" , "membership.h" , "memory.h" , "paths.h" , "pcreposix.h" , "pcre2posix.h" , "poll.h" , "process.h" , "pthread.h" , "pwd.h" , "readpassphrase.h" , "regex.h" , "signal.h" , "spawn.h" , "stdarg.h" , "stdint.h" , "stdlib.h" , "string.h" , "strings.h" , "sys/acl.h" , "sys/cdefs.h" , "sys/extattr.h" , "sys/ioctl.h" , "sys/mkdev.h" , "sys/mount.h" , "sys/param.h" , "sys/poll.h" , "sys/queue.h" , "sys/richacl.h" , "sys/select.h" , "sys/stat.h" , "sys/statfs.h" , "sys/statvfs.h" , "sys/sysmacros.h" , "sys/time.h" , "sys/utime.h" , "sys/utsname.h" , "sys/vfs.h" , "sys/wait.h" , "sys/xattr.h" , "time.h" , "unistd.h" , "utime.h" , "wchar.h" , "wctype.h" , "windows.h" , "wincrypt.h" , "winioctl.h" , "bcrypt.h" , "mbedtls/aes.h" , "mbedtls/md.h" , "mbedtls/pkcs5.h" , "md5.h" , "rmd160.h" , "sha1.h" , "sha2.h" , "ripemd.h" , "sha.h" , "sha256.h" , "sha512.h" , "CommonCrypto/CommonDigest.h" , "mbedtls/md5.h" , "mbedtls/ripemd160.h" , "mbedtls/sha1.h" , "mbedtls/sha256.h" , "mbedtls/sha512.h" , "nettle/md5.h" , "nettle/ripemd160.h" , "nettle/sha.h" , "openssl/evp.h" , "openssl/opensslv.h" , "iconv.h" , "localcharset.h" , "libxml/xmlreader.h" , "libxml/xmlwriter.h" , "assert.h" , "sys/richacl.h" , "getopt.h" ] ] , [ "defines1" , { "type": "++" , "$1": [ [ ["__LIBARCHIVE_CONFIG_H_INCLUDED", 1] , ["HAVE_LIBZ", {"type": "var", "name": "ENABLE_ZLIB"}] , ["HAVE_LIBBZ2", {"type": "var", "name": "ENABLE_BZip2"}] , ["HAVE_LIBLZMA", {"type": "var", "name": "ENABLE_LZMA"}] , ["HAVE_LIBLZO2", {"type": "var", "name": "ENABLE_LZO"}] , ["HAVE_LIBLZ4", {"type": "var", "name": "ENABLE_LZ4"}] , [ "HAVE_LIBMBEDCRYPTO" , {"type": "var", "name": "ENABLE_MBEDTLS"} ] , ["HAVE_LIBNETTLE", {"type": "var", "name": "ENABLE_NETTLE"}] , ["HAVE_LIBMD", {"type": "var", "name": "ENABLE_LIBMD"}] , ["HAVE_LIBCRYPTO", {"type": "var", "name": "ENABLE_OPENSSL"}] , ["HAVE_LIBXML2", {"type": "var", "name": "ENABLE_LIBXML2"}] , ["HAVE_LIBEXPAT", {"type": "var", "name": "ENABLE_EXPAT"}] , [ "HAVE_LIBPCREPOSIX" , {"type": "var", "name": "ENABLE_PCREPOSIX"} ] , [ "HAVE_LIBPCRE2POSIX" , {"type": "var", "name": "ENABLE_PCRE2POSIX"} ] , ["HAVE_LIBPCRE", {"type": "var", "name": "ENABLE_PCRE"}] , ["HAVE_LIBPCRE2", {"type": "var", "name": "ENABLE_PCRE2"}] , ["HAVE_LIBGCC", {"type": "var", "name": "ENABLE_LIBGCC"}] , ["HAVE_LIBACL", {"type": "var", "name": "ENABLE_ACL"}] , ["HAVE_LIBRICHACL", {"type": "var", "name": "ENABLE_RICHACL"}] ] , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ ["interix", [["_ALL_SOURCE", 1]]] , [ "solaris" , [["_POSIX_PTHREAD_SEMANTICS", 1], ["__EXTENSIONS__", 1]] ] , ["linux", [["_GNU_SOURCE", 1]]] , ["nonstop", [["_TANDEM_SOURCE", 1]]] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ ["darwin", [["ARCHIVE_XATTR_DARWIN", 1]]] , ["bsd", [["ARCHIVE_XATTR_FREEBSD", 1]]] , ["linux", [["ARCHIVE_XATTR_LINUX", 1]]] ] } } , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "darwin" , { "type": "if" , "cond": { "type": "or" , "$1": [ {"type": "var", "name": "ENABLE_ACL"} , {"type": "var", "name": "ENABLE_RICHACL"} ] } , "then": [["ARCHIVE_ACL_DARWIN", 1]] } ] , [ "bsd" , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_NFS4"} , "then": [["ARCHIVE_ACL_FREEBSD_NFS4"]] , "else": [["ARCHIVE_ACL_FREEBSD"]] } } ] , [ "sunos" , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_NFS4"} , "then": [["ARCHIVE_ACL_SUNOS_NFS4"]] , "else": [["ARCHIVE_ACL_SUNOS"]] } } ] , [ "linux" , [ [ "ARCHIVE_ACL_LIBACL" , {"type": "var", "name": "ENABLE_ACL"} ] , [ "ARCHIVE_ACL_LIBRICHACL" , {"type": "var", "name": "ENABLE_RICHACL"} ] ] ] ] } ] } ] , [ "have_cfile" , { "type": "++" , "$1": [ [ ["HAVE_DIRENT_H", "dirent.h"] , ["HAVE_SYS_NDIR_H", "sys/ndir.h"] , ["HAVE_NDIR_H", "ndir.h"] , ["HAVE_SYS_DIR_H", "sys/dir.h"] , ["HAVE_SYS_TYPES_H", "sys/types.h"] , ["HAVE_ACL_LIBACL_H", "acl/libacl.h"] , ["HAVE_ATTR_XATTR_H", "attr/xattr.h"] , ["HAVE_CTYPE_H", "ctype.h"] , ["HAVE_COPYFILE_H", "copyfile.h"] , ["HAVE_DIRECT_H", "direct.h"] , ["HAVE_DLFCN_H", "dlfcn.h"] , ["HAVE_ERRNO_H", "errno.h"] , ["HAVE_EXT2FS_EXT2_FS_H", "ext2fs/ext2_fs.h"] , ["HAVE_FCNTL_H", "fcntl.h"] , ["HAVE_FNMATCH_H", "fnmatch.h"] , ["HAVE_GRP_H", "grp.h"] , ["HAVE_INTTYPES_H", "inttypes.h"] , ["HAVE_IO_H", "io.h"] , ["HAVE_LANGINFO_H", "langinfo.h"] , ["HAVE_LIMITS_H", "limits.h"] , ["HAVE_LINUX_TYPES_H", "linux/types.h"] , ["HAVE_LINUX_FIEMAP_H", "linux/fiemap.h"] , ["HAVE_LINUX_FS_H", "linux/fs.h"] , ["HAVE_LINUX_MAGIC_H", "linux/magic.h"] , ["HAVE_LOCALE_H", "locale.h"] , ["HAVE_MEMBERSHIP_H", "membership.h"] , ["HAVE_MEMORY_H", "memory.h"] , ["HAVE_PATHS_H", "paths.h"] , ["HAVE_POLL_H", "poll.h"] , ["HAVE_PROCESS_H", "process.h"] , ["HAVE_PTHREAD_H", "pthread.h"] , ["HAVE_PWD_H", "pwd.h"] , ["HAVE_READPASSPHRASE_H", "readpassphrase.h"] , ["HAVE_REGEX_H", "regex.h"] , ["HAVE_SIGNAL_H", "signal.h"] , ["HAVE_SPAWN_H", "spawn.h"] , ["HAVE_STDARG_H", "stdarg.h"] , ["HAVE_STDINT_H", "stdint.h"] , ["HAVE_STDLIB_H", "stdlib.h"] , ["HAVE_STRING_H", "string.h"] , ["HAVE_STRINGS_H", "strings.h"] , ["HAVE_SYS_ACL_H", "sys/acl.h"] , ["HAVE_SYS_CDEFS_H", "sys/cdefs.h"] , ["HAVE_SYS_EXTATTR_H", "sys/extattr.h"] , ["HAVE_SYS_IOCTL_H", "sys/ioctl.h"] , ["HAVE_SYS_MKDEV_H", "sys/mkdev.h"] , ["HAVE_SYS_MOUNT_H", "sys/mount.h"] , ["HAVE_SYS_PARAM_H", "sys/param.h"] , ["HAVE_SYS_POLL_H", "sys/poll.h"] , ["HAVE_SYS_QUEUE_H", "sys/queue.h"] , ["HAVE_SYS_RICHACL_H", "sys/richacl.h"] , ["HAVE_SYS_SELECT_H", "sys/select.h"] , ["HAVE_SYS_STAT_H", "sys/stat.h"] , ["HAVE_SYS_STATFS_H", "sys/statfs.h"] , ["HAVE_SYS_STATVFS_H", "sys/statvfs.h"] , ["HAVE_SYS_SYSMACROS_H", "sys/sysmacros.h"] , ["HAVE_SYS_TIME_H", "sys/time.h"] , ["HAVE_SYS_UTIME_H", "sys/utime.h"] , ["HAVE_SYS_UTSNAME_H", "sys/utsname.h"] , ["HAVE_SYS_VFS_H", "sys/vfs.h"] , ["HAVE_SYS_WAIT_H", "sys/wait.h"] , ["HAVE_SYS_XATTR_H", "sys/xattr.h"] , ["HAVE_TIME_H", "time.h"] , ["HAVE_UNISTD_H", "unistd.h"] , ["HAVE_UTIME_H", "utime.h"] , ["HAVE_WCHAR_H", "wchar.h"] , ["HAVE_WCTYPE_H", "wctype.h"] , ["HAVE_WINDOWS_H", "windows.h"] , ["HAVE_WINCRYPT_H", "wincrypt.h"] , ["HAVE_WINIOCTL_H", "winioctl.h"] ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZLIB"} , "then": [["HAVE_ZLIB_H", "zlib.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": [["HAVE_OPENSSL_EVP_H", "openssl/evp.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_BZip2"} , "then": [["HAVE_BZLIB_H", "bzlib.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": [["HAVE_LZMA_H", "lzma.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZO"} , "then": [ ["HAVE_LZO_LZOCONF_H", "lzo/lzoconf.h"] , ["HAVE_LZO_LZO1X_H", "lzo/lzo1x.h"] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBB2"} , "then": [["HAVE_BLAKE2_H", "blake.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZ4"} , "then": [["HAVE_LZ4_H", "lz4.h"], ["HAVE_LZ4HC_H", "lz4hc.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZSTD"} , "then": [["HAVE_ZSTD_H", "zstd.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCREPOSIX"} , "then": [["HAVE_PCREPOSIX_H", "pcreposix.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE2POSIX"} , "then": [["HAVE_PCRE2POSIX_H", "pcre2posix.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_CNG"} , "then": [["HAVE_BCRYPT_H", "bcrypt.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_MBEDTLS"} , "then": [ ["HAVE_MBEDTLS_AES_H", "mbedtls/aes.h"] , ["HAVE_MBEDTLS_MD_H", "mbedtls/md.h"] , ["HAVE_MBEDTLS_PKCS5_H", "mbedtls/pkcs5.h"] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_NETTLE"} , "then": [ ["HAVE_NETTLE_AES_H", "nettle/aes.h"] , ["HAVE_NETTLE_HMAC_H", "nettle/hmac.h"] , ["HAVE_NETTLE_MD5_H", "nettle/md5.h"] , ["HAVE_NETTLE_PBKDF2_H", "nettle/pbkdf2.h"] , ["HAVE_NETTLE_RIPEMD160_H", "nettle/ripemd160.h"] , ["HAVE_NETTLE_SHA_H", "nettle/sha.h"] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ICONV"} , "then": [ ["HAVE_ICONV_H", "iconv.h"] , ["HAVE_LOCALCHARSET_H", "localcharset.h"] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBXML2"} , "then": [ ["HAVE_LIBXML_XMLREADER_H", "libxml/xmlreader.h"] , ["HAVE_LIBXML_XMLWRITER_H", "libxml/xmlwriter.h"] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_EXPAT"} , "then": [["HAVE_EXPAT_H", "expat.h"]] } ] } ] , [ "have_csymbol" , { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_ZLIB"} , { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "cygwin" } , "then": false , "else": true } ] } , "then": [["ZLIB_WINAPI", ["ZLIB_WINAPI", ["zlib.h"]]]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBB2"} , "then": [["HAVE_LIBB2", ["blake2sp_init", "blake2.h"]]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZSTD"} , "then": [ ["HAVE_LIBZSTD", ["ZSTD_decompressStream", ["zstd.h"]]] , [ "HAVE_LIBZSTD_compressStream" , ["ZSTD_compressStream", ["zstd.h"]] ] ] } , [ [ "HAVE_WORKING_EXT2_IOC_GETFLAGS" , ["EXT2_IOC_GETFLAGS", ["sys/ioctl.h", "ext2fs/ext2_fs.h"]] ] , [ "HAVE_WORKING_FS_IOC_GETFLAGS" , ["FS_IOC_GETFLAGS", ["sys/ioctl.h", "linux/fs.h"]] ] , [ "SAFE_TO_DEFINE_EXTENSIONS" , ["__EXTENSIONS__", {"type": "var", "name": "sys_hdrs"}] ] ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ICONV"} , "then": [ [ "HAVE_LOCALE_CHARSET" , ["locale_charset", {"type": "var", "name": "sys_hdrs"}] ] ] } , [ ["HAVE__CrtSetReportMode", ["_CrtSetReportMode", ["crtdbg.h"]]] , [ "HAVE_ARC4RANDOM_BUF" , ["arc4random_buf", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_CHFLAGS" , ["chflags", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_CHOWN", ["chown", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_CHROOT" , ["chroot", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_CTIME_R" , ["ctime_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FCHDIR" , ["fchdir", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FCHFLAGS" , ["fchflags", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FCHMOD" , ["fchmod", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FCHOWN" , ["fchown", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_FCNTL", ["fcntl", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_FDOPENDIR" , ["fdopendir", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FNMATCH" , ["fnmatch", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_FORK", ["fork", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_FSTAT", ["fstat", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_FSTATAT" , ["fstatat", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSTATFS" , ["fstatfs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSTATVFS" , ["fstatvfs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FTRUNCATE" , ["ftruncate", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FUTIMENS" , ["futimens", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FUTIMES" , ["futimes", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FUTIMESAT" , ["futimesat", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETEUID" , ["geteuid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETGRGID_R" , ["getgrgid_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETGRNAM_R" , ["getgrnam_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETGRNAM_R" , ["getgrnam_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPWNAM_R" , ["getpwnam_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPWUID_R" , ["getpwuid_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETPID" , ["getpid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETVFSBYNAME" , ["getvfsbyname", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GMTIME_R" , ["gmtime_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LCHFLAGS" , ["lchflags", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LCHOWN" , ["lchown", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_LINK", ["link", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_LINKAT" , ["linkat", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LOCALTIME_R" , ["localtime_r", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_LSTAT", ["lstat", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_LUTIMES" , ["lutimes", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MBRTOWC" , ["mbrtowc", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MEMMOVE" , ["memmove", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_MKDIR", ["mkdir", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_MKFIFO" , ["mkfifo", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_MKNOD", ["mknod", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_MKSTEMP" , ["mkstemp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_NL_LANGINFO" , ["nl_langinfo", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_OPENAT" , ["openat", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_PIPE", ["pipe", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_POLL", ["poll", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_POSIX_SPAWNP" , ["posix_spawnp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_READLINK" , ["readlink", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_READPASSPHRASE" , ["readpassphrase", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SELECT" , ["select", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETENV" , ["setenv", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETLOCALE" , ["setlocale", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SIGACTION" , ["sigaction", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STATFS" , ["statfs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STATVFS" , ["statvfs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRCHR" , ["strchr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRDUP" , ["strdup", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRDUP" , ["strdup", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRERROR" , ["strerror", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRNCPY_S" , ["strncpy_s", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRNLEN" , ["strnlen", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRRCHR" , ["strrchr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRUCT_STATFS" , ["statfs", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SYMLINK" , ["symlink", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SYSCONF" , ["sysconf", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_TIMEGM" , ["timegm", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_TZSET", ["tzset", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_UNLINKAT" , ["unlinkat", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_UNSETENV" , ["unsetenv", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_UTIME", ["utime", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_UTIMES" , ["utimes", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_UTIMENSAT" , ["utimensat", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_VFORK", ["vfork", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_WCRTOMB" , ["wcrtomb", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WCSCMP" , ["wcscmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WCSCPY" , ["wcscpy", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WCSLEN" , ["wcslen", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WCTOMB" , ["wctomb", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE__CTIME64_S" , ["_ctime64_s", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE__FSEEKI64" , ["_fseeki64", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE__GET_TIMEZONE" , ["_get_timezone", {"type": "var", "name": "sys_hdrs"}] ] , ["HAVE_CTIME_S", ["ctime_s", ["time.h"]]] , ["HAVE_GMTIME_S", ["gmtime_s", ["time.h"]]] , ["HAVE_LOCALTIME_S", ["localtime_s", ["time.h"]]] , ["HAVE__MKGMTIME", ["_mkgmtime", ["time.h"]]] ] , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "linux" } , "then": [] , "else": [ [ "HAVE_LCHMOD" , ["lchmod", {"type": "var", "name": "sys_hdrs"}] ] ] } , [ [ "HAVE_CYGWIN_CONV_PATH" , ["cygwin_conv_path", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSEEKO" , ["fseeko", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRERROR_R" , ["strerror_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_DECL_STRERROR_R" , ["strerror_r", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_STRFTIME" , ["strftime", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_VPRINTF" , ["vprintf", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WMEMCMP" , ["wmemcmp", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WMEMCPY" , ["wmemcpy", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_WMEMMOVE" , ["wmemmove", {"type": "var", "name": "sys_hdrs"}] ] ] , [ [ "HAVE_STRUCT_VFSCONF" , ["vfsconf", ["sys/typesh", "sys/mount.h"]] ] , [ "HAVE_STRUCT_XVFSCONF" , ["xvfsconf", ["sys/typesh", "sys/mount.h"]] ] , [ "HAVE_STRUCT_STATFS" , ["statfs", ["sys/typesh", "sys/mount.h"]] ] , ["HAVE_READDIR_R", ["readdir_r", ["dirent.h"]]] , ["HAVE_DIRFD", ["dirfd", ["dirent.h"]]] , ["HAVE_READLINKAT", ["readlinkat", ["fcntl.h", "unistd.h"]]] , ["HAVE_GETOPT_OPTRESET", ["optreset", ["getopt.h"]]] , ["MAJOR_IN_MKDEV", ["major", ["sys/mkdev.h"]]] , ["MAJOR_IN_SYSMACROS", ["major", ["sys/sysmacros.h"]]] ] , [ ["HAVE_EFTYPE", ["EFTYPE", ["errno.h"]]] , ["HAVE_EILSEQ", ["EILSEQ", ["errno.h"]]] , ["HAVE_D_MD_ORDER", ["D_MD_ORDER", ["langinfo.h"]]] , [ "HAVE_DECL_INT32_MAX" , ["INT32_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_INT32_MIN" , ["INT32_MIN", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_INT64_MAX" , ["INT64_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_INT64_MIN" , ["INT64_MIN", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_INTMAX_MAX" , ["INTMAX_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_INTMAX_MIN" , ["INTMAX_MIN", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_UINT32_MAX" , ["UINT32_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_UINT64_MAX" , ["UINT64_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_UINTMAX_MAX" , ["UINTMAX_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , [ "HAVE_DECL_SIZE_MAX" , ["SIZE_MAX", ["limits.h", "stdint.h", "inttypes.h"]] ] , ["HAVE_DECL_SSIZE_MAX", ["SSIZE_MAX", ["limits.h"]]] ] , [ [ "HAVE_STRUCT_TM_TM_GMTOFF" , [ "((struct tm*)NULL)->tm_gmtoff" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_TM___TM_GMTOFF" , [ "((struct tm*)NULL)->__tm_gmtoff" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STATFS_F_IOSIZE" , [ "((struct statfs*)NULL)->f_iosize" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STATFS_F_NAMEMAX" , [ "((struct statfs*)NULL)->f_namemax" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STATFS_F_IOSIZE" , [ "((struct statfs*)NULL)->f_iosize" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_BIRTHTIME" , [ "((struct stat*)NULL)->st_birthtime" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_BIRTHTIMESPEC_TV_NSEC" , [ "((struct stat*)NULL)->st_birthtimespec.tv_nsec" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC" , [ "((struct stat*)NULL)->st_mtimespec.tv_nsec" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC" , [ "((struct stat*)NULL)->st_mtim.tv_nsec" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_MTIME_N" , [ "((struct stat*)NULL)->st_mtime_n" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_UMTIME" , [ "((struct stat*)NULL)->st_umtime" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_MTIME_USEC" , [ "((struct stat*)NULL)->st_mtime_usec" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_BLKSIZE" , [ "((struct stat*)NULL)->st_blksize" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STAT_ST_FLAGS" , [ "((struct stat*)NULL)->st_flags" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_STATVFS_F_IOSIZE" , [ "((struct statvfs*)NULL)->f_iosize" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_SYS_TIME_H" , [ "((struct tm*)NULL)->tm_sec" , {"type": "var", "name": "sys_hdrs"} ] ] ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": [ [ "HAVE_PKCS5_PBKDF2_HMAC_SHA1" , [ "PKCS5_PBKDF2_HMAC_SHA1" , {"type": "var", "name": "sys_hdrs"} ] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": [ [ "HAVE_LZMA_STREAM_ENCODER_MT" , ["lzma_stream_encoder_mt", ["lzma.h"]] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": [ [ "HAVE_DECL_EXTATTR_NAMESPACE_USER" , [ "EXTATTR_NAMESPACE_USER" , ["sys/types.h", "sys/extattr.h"] ] ] , [ "HAVE_DECL_XATTR_NOFOLLOW" , ["XATTR_NOFOLLOW", ["sys/xattr.h"]] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "darwin" , [ [ "HAVE_FGETXATTR" , ["fgetxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FLISTXATTR" , ["flistxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSETXATTR" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETXATTR" , ["getxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LISTXATTR" , ["listxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_SETXATTR" , ["setxattr", {"type": "var", "name": "sys_hdrs"}] ] ] ] , [ "bsd" , [ [ "HAVE_EXTATTR_GET_FD" , [ "extattr_get_fd" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_GET_FILE" , [ "extattr_get_file" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_GET_LINK" , [ "extattr_get_link" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_LIST_FD" , [ "extattr_list_fd" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_LIST_FILE" , [ "extattr_list_file" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_LIST_LINK" , [ "extattr_list_link" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_SET_FD" , [ "extattr_set_fd" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_EXTATTR_SET_LINK" , [ "extattr_set_link" , {"type": "var", "name": "sys_hdrs"} ] ] ] ] , [ "linux" , [ [ "HAVE_FGETXATTR" , ["fgetxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FLISTXATTR" , ["flistxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_FSETXATTR" , ["fsetxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_GETXATTR" , ["getxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LGETXATTR" , ["lgetxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LISTXATTR" , ["listxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LLISTXATTR" , ["llistxattr", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_LSETXATTR" , ["lsetxattr", {"type": "var", "name": "sys_hdrs"}] ] ] ] ] } } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ACL"} , "then": [ ["HAVE_ACL", ["acl", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_FACL", ["facl", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_ACL_T" , ["sizeof(acl_t)", ["sys/types.h", "sys.acl.h"]] ] , [ "HAVE_ACL_ENTRY_T" , ["sizeof(acl_entry_t)", ["sys/types.h", "sys.acl.h"]] ] , [ "HAVE_ACL_PERMSET_T" , ["sizeof(acl_permset_t)", ["sys/types.h", "sys.acl.h"]] ] , [ "HAVE_ACL_TAG_T" , ["sizeof(acl_tag_t)", ["sys/types.h", "sys.acl.h"]] ] , ["HAVE_ACLENT_T", ["sizeof(aclent_t)", ["sys/acl.h"]]] , ["HAVE_DECL_GETACL", ["GETACL", ["sys/acl.h"]]] , ["HAVE_DECL_GETACLCNT", ["GETACLCNT", ["sys/acl.h"]]] , ["HAVE_DECL_SETACL", ["SETACL", ["sys/acl.h"]]] , ["HAVE_ACE_T", ["sizeof(ace_t)", ["sys/acl.h"]]] , ["HAVE_DECL_ACE_GETACL", ["ACE_GETACL", ["sys/acl.h"]]] , ["HAVE_DECL_ACE_GETACLCNT", ["ACE_GETACLCNT", ["sys/acl.h"]]] , ["HAVE_DECL_ACE_SETACL", ["ACE_SETACL", ["sys/acl.h"]]] , [ "HAVE_ACL_ADD_PERM" , ["acl_add_perm", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_CLEAR_PERMS" , ["acl_clear_perms", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_CREATE_ENTRY" , ["acl_create_entry", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_DELETE_DEF_FILE" , [ "acl_delete_def_file" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_ACL_FREE" , ["acl_free", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_ENTRY" , ["acl_get_entry", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_FD" , ["acl_get_fd", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_FILE" , ["acl_get_file", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_PERMSET" , ["acl_get_permset", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_QUALIFIER" , ["acl_get_qualifier", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_TAG_TYPE" , ["acl_get_tag_type", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_INIT" , ["acl_init", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_FD" , ["acl_set_fd", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_FILE" , ["acl_set_file", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_QUALIFIER" , ["acl_set_qualifier", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_TAG_TYPE" , ["acl_set_tag_type", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_PERM" , ["acl_get_perm", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_ADD_FLAG_NP" , ["acl_add_flag_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_CLEAR_FLAGS_NP" , ["acl_clear_flags_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_BRAND_NP" , ["acl_get_brand_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_ENTRY_TYPE_NP" , [ "acl_get_entry_type_np" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_ACL_GET_FLAG_NP" , ["acl_get_flag_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_FLAGSET_NP" , ["acl_get_flagset_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_FD_NP" , ["acl_get_fd_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_LINK_NP" , ["acl_get_link_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_GET_PERM_NP" , ["acl_get_perm_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_IS_TRIVIAL_NP" , ["acl_is_trivial_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_ENTRY_TYPE_NP" , [ "acl_set_entry_type_np" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_ACL_SET_FD_NP" , ["acl_set_fd_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_ACL_SET_LINK_NP" , ["acl_set_link_np", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MBR_GID_TO_UUID" , ["mbr_gid_to_uuid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MBR_UID_TO_UUID" , ["mbr_uid_to_uuid", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_MBR_UUID_TO_ID" , ["mbr_uuid_to_id", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_DECL_ACL_TYPE_EXTENDED" , ["ACL_TYPE_EXTENDED", ["sys/types.h", "sys/acl.h"]] ] , [ "HAVE_DECL_ACL_SYNCHRONIZE" , ["ACL_SYNCHRONIZE", ["sys/types.h", "sys/acl.h"]] ] , ["HAVE_DECL_ACL_TYPE_NFS4", ["ACL_TYPE_NFS4", ["sys/acl.h"]]] , ["HAVE_DECL_ACL_USER", ["ACL_USER", ["sys/acl.h"]]] , [ "HAVE_STRUCT_RICHACE" , [ "((struct richace*)NULL)->e_type" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_STRUCT_RICHACL" , [ "((struct richacl*)NULL)->a_flags" , {"type": "var", "name": "sys_hdrs"} ] ] , [ "HAVE_RICHACL_ALLOC" , ["richacl_alloc", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_EQUIV_MODE" , ["richacl_equiv_mode", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_FREE" , ["richacl_free", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_GET_FD" , ["richacl_get_fd", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_GET_FILE" , ["richacl_get_file", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_SET_FD" , ["richacl_set_fd", {"type": "var", "name": "sys_hdrs"}] ] , [ "HAVE_RICHACL_SET_FILE" , ["richacl_set_file", {"type": "var", "name": "sys_hdrs"}] ] ] } , [ ["ARCHIVE_CRYPTO_MD5_LIBC", ["archive_md5_ctx", ["md5.h"]]] , [ "ARCHIVE_CRYPTO_RMD160_LIBC" , ["archive_rmd160_ctx", ["rmd160.h"]] ] , ["ARCHIVE_CRYPTO_SHA1_LIBC", ["archive_sha1_ctx", ["sha1.h"]]] , [ "ARCHIVE_CRYPTO_SHA256_LIBC" , ["archive_sha256_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_LIBC2" , ["archive_sha256_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_LIBC3" , ["archive_sha256_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_LIBC" , ["archive_sha384_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_LIBC2" , ["archive_sha384_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_LIBC3" , ["archive_sha384_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_LIBC" , ["archive_sha512_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_LIBC2" , ["archive_sha512_ctx", ["sha2.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_LIBC3" , ["archive_sha512_ctx", ["sha2.h"]] ] ] , [ [ "ARCHIVE_CRYPTO_MD5_LIBSYSTEM" , ["archive_md5_ctx", ["CommonCrypto/CommonDigest.h"]] ] , [ "ARCHIVE_CRYPTO_SHA1_LIBSYSTEM" , ["archive_sha1_ctx", ["CommonCrypto/CommonDigest.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_LIBSYSTEM" , ["archive_sha256_ctx", ["CommonCrypto/CommonDigest.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_LIBSYSTEM" , ["archive_sha384_ctx", ["CommonCrypto/CommonDigest.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_LIBSYSTEM" , ["archive_sha512_ctx", ["CommonCrypto/CommonDigest.h"]] ] ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_MBEDTLS"} , "then": [ [ "ARCHIVE_CRYPTO_MD5_MBEDTLS" , ["archive_md5_ctx", ["mbedtls/md5.h"]] ] , [ "ARCHIVE_CRYPTO_RMD160_MBEDTLS" , ["archive_rmd160_ctx", ["mbedtls/ripemd160.h"]] ] , [ "ARCHIVE_CRYPTO_SHA1_MBEDTLS" , ["archive_sha1_ctx", ["mbedtls/sha1.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_MBEDTLS" , ["archive_sha256_ctx", ["mbedtls/sha256.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_MBEDTLS" , ["archive_sha384_ctx", ["mbedtls/sha512.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_MBEDTLS" , ["archive_sha512_ctx", ["mbedtls/sha512.h"]] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_NETTLE"} , "then": [ [ "ARCHIVE_CRYPTO_MD5_NETTLE" , ["archive_md5_ctx", ["nettle/md5.h"]] ] , [ "ARCHIVE_CRYPTO_RMD160_NETTLE" , ["archive_rmd160_ctx", ["nettle/ripemd160.h"]] ] , [ "ARCHIVE_CRYPTO_SHA1_NETTLE" , ["archive_sha1_ctx", ["nettle/sha.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_NETTLE" , ["archive_sha256_ctx", ["nettle/sha.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_NETTLE" , ["archive_sha384_ctx", ["nettle/sha.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_NETTLE" , ["archive_sha512_ctx", ["nettle/sha.h"]] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": [ [ "ARCHIVE_CRYPTO_MD5_OPENSSL" , [ "archive_md5_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] , [ "ARCHIVE_CRYPTO_RMD160_OPENSSL" , [ "archive_rmd160_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] , [ "ARCHIVE_CRYPTO_SHA1_OPENSSL" , [ "archive_sha1_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] , [ "ARCHIVE_CRYPTO_SHA256_OPENSSL" , [ "archive_sha256_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] , [ "ARCHIVE_CRYPTO_SHA384_OPENSSL" , [ "archive_sha384_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] , [ "ARCHIVE_CRYPTO_SHA512_OPENSSL" , [ "archive_sha512_ctx" , ["openssl/evp.h", "openssl/opensslv.h"] ] ] ] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBMD"} , "then": [ ["ARCHIVE_CRYPTO_MD5_LIBMD", ["archive_md5_ctx", ["md5.h"]]] , [ "ARCHIVE_CRYPTO_RMD160_LIBMD" , ["archive_rmd160_ctx", ["ripemd.h"]] ] , [ "ARCHIVE_CRYPTO_SHA1_LIBMD" , ["archive_sha1_ctx", ["sha.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_LIBMD" , ["archive_sha256_ctx", ["sha256.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_LIBMD" , ["archive_sha512_ctx", ["sha512.h"]] ] ] } , { "type": "if" , "cond": { "type": "and" , "$1": [ { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "COMPILER_FAMILY"} , "$2": "cygwin" } , "then": false , "else": true } ] } , "then": [ [ "ARCHIVE_CRYPTO_MD5_WIN" , ["CALG_MD5", ["windows.h", "wincrypt.h"]] ] , [ "ARCHIVE_CRYPTO_SHA1_WIN" , ["CALG_SHA1", ["windows.h", "wincrypt.h"]] ] , [ "ARCHIVE_CRYPTO_SHA256_WIN" , ["CALG_SHA_256", ["windows.h", "wincrypt.h"]] ] , [ "ARCHIVE_CRYPTO_SHA384_WIN" , ["CALG_SHA_384", ["windows.h", "wincrypt.h"]] ] , [ "ARCHIVE_CRYPTO_SHA512_WIN" , ["CALG_SHA_512", ["windows.h", "wincrypt.h"]] ] ] } ] } ] , [ "have_ctype" , [ ["HAVE_INTMAX_T", "intmax_t"] , ["HAVE_UINTMAX_T", "uintmax_t"] , ["HAVE_UNSIGNED_LONG_LONG_INT", "unsigned long long"] , ["HAVE_LONG_LONG_INT", "long long int"] , ["HAVE_WCHAR_T", "wchar_t"] ] ] , [ "size_ctype" , [ ["SIZEOF_INT", ["int", [1, 2, 4, 8, 16]]] , ["SIZEOF_LONG", ["long", [1, 2, 4, 8, 16]]] , ["SIZEOF_WCHAR_T", ["wchar_t", [1, 2, 4, 8, 16]]] ] ] , [ "defines" , { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": [] , "else": [["HAVE_LZMA_STREAM_ENCODER_MT", "0"]] } , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } , "then": [ ["id_t", "unsigned long"] , ["mode_t", "int"] , ["off_t", "long long"] , { "type": "case*" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": [["mingw", [["gid_t", "short"], ["uid_t", "short"]]]] , "default": [["gid_t", "int"], ["uid_t", "int"]] } ] } ] } ] ] , "body": { "type": "env" , "vars": [ "defines1" , "have_cfile" , "have_csymbol" , "have_ctype" , "size_ctype" , "defines" ] } } } , "config_header": { "type": ["@", "rules", "CC/auto", "config"] , "name": ["config.h"] , "guard": ["INCLUDE_archive_config_h__"] , "deps": [ ["src", "archive_acl"] , ["src", "archive_bzip2"] , ["src", "archive_crypt"] , ["src", "archive_crypto"] , ["src", "archive_libb2"] , ["src", "archive_libgcc"] , ["src", "archive_libmd"] , ["src", "archive_libxml2"] , ["src", "archive_lz4"] , ["src", "archive_lzma"] , ["src", "archive_lzo"] , ["src", "archive_lzstd"] , ["src", "archive_mbedtls"] , ["src", "archive_nettle"] , ["src", "archive_pcre"] , ["src", "archive_pcre2"] , ["src", "archive_pcre2posix"] , ["src", "archive_pcreposix"] , ["src", "archive_pthread"] , ["src", "archive_regex"] , ["src", "archive_richacl"] , ["src", "archive_xattr"] , ["src", "archive_zlib"] ] } , "libarchive_csources": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "archive_acl.c" , "archive_blake2s_ref.c" , "archive_blake2sp_ref.c" , "archive_check_magic.c" , "archive_cmdline.c" , "archive_cryptor.c" , "archive_digest.c" , "archive_disk_acl_darwin.c" , "archive_disk_acl_freebsd.c" , "archive_disk_acl_linux.c" , "archive_disk_acl_sunos.c" , "archive_entry.c" , "archive_entry_copy_bhfi.c" , "archive_entry_copy_stat.c" , "archive_entry_link_resolver.c" , "archive_entry_sparse.c" , "archive_entry_stat.c" , "archive_entry_strmode.c" , "archive_entry_xattr.c" , "archive_getdate.c" , "archive_hmac.c" , "archive_match.c" , "archive_options.c" , "archive_pack_dev.c" , "archive_pathmatch.c" , "archive_ppmd7.c" , "archive_ppmd8.c" , "archive_random.c" , "archive_rb.c" , "archive_read.c" , "archive_read_add_passphrase.c" , "archive_read_append_filter.c" , "archive_read_data_into_fd.c" , "archive_read_disk_entry_from_file.c" , "archive_read_disk_posix.c" , "archive_read_disk_set_standard_lookup.c" , "archive_read_disk_windows.c" , "archive_read_extract.c" , "archive_read_extract2.c" , "archive_read_open_fd.c" , "archive_read_open_file.c" , "archive_read_open_filename.c" , "archive_read_open_memory.c" , "archive_read_set_format.c" , "archive_read_set_options.c" , "archive_read_support_filter_all.c" , "archive_read_support_filter_by_code.c" , "archive_read_support_filter_bzip2.c" , "archive_read_support_filter_compress.c" , "archive_read_support_filter_grzip.c" , "archive_read_support_filter_gzip.c" , "archive_read_support_filter_lrzip.c" , "archive_read_support_filter_lz4.c" , "archive_read_support_filter_lzop.c" , "archive_read_support_filter_none.c" , "archive_read_support_filter_program.c" , "archive_read_support_filter_rpm.c" , "archive_read_support_filter_uu.c" , "archive_read_support_filter_xz.c" , "archive_read_support_filter_zstd.c" , "archive_read_support_format_7zip.c" , "archive_read_support_format_all.c" , "archive_read_support_format_ar.c" , "archive_read_support_format_by_code.c" , "archive_read_support_format_cab.c" , "archive_read_support_format_cpio.c" , "archive_read_support_format_empty.c" , "archive_read_support_format_iso9660.c" , "archive_read_support_format_lha.c" , "archive_read_support_format_mtree.c" , "archive_read_support_format_rar.c" , "archive_read_support_format_rar5.c" , "archive_read_support_format_raw.c" , "archive_read_support_format_tar.c" , "archive_read_support_format_warc.c" , "archive_read_support_format_xar.c" , "archive_read_support_format_zip.c" , "archive_string.c" , "archive_string_sprintf.c" , "archive_util.c" , "archive_version_details.c" , "archive_virtual.c" , "archive_windows.c" , "archive_write.c" , "archive_write_add_filter.c" , "archive_write_add_filter_b64encode.c" , "archive_write_add_filter_by_name.c" , "archive_write_add_filter_bzip2.c" , "archive_write_add_filter_compress.c" , "archive_write_add_filter_grzip.c" , "archive_write_add_filter_gzip.c" , "archive_write_add_filter_lrzip.c" , "archive_write_add_filter_lz4.c" , "archive_write_add_filter_lzop.c" , "archive_write_add_filter_none.c" , "archive_write_add_filter_program.c" , "archive_write_add_filter_uuencode.c" , "archive_write_add_filter_xz.c" , "archive_write_add_filter_zstd.c" , "archive_write_disk_posix.c" , "archive_write_disk_set_standard_lookup.c" , "archive_write_disk_windows.c" , "archive_write_open_fd.c" , "archive_write_open_file.c" , "archive_write_open_filename.c" , "archive_write_open_memory.c" , "archive_write_set_format.c" , "archive_write_set_format_7zip.c" , "archive_write_set_format_ar.c" , "archive_write_set_format_by_name.c" , "archive_write_set_format_cpio.c" , "archive_write_set_format_cpio_binary.c" , "archive_write_set_format_cpio_newc.c" , "archive_write_set_format_cpio_odc.c" , "archive_write_set_format_filter_by_ext.c" , "archive_write_set_format_gnutar.c" , "archive_write_set_format_iso9660.c" , "archive_write_set_format_mtree.c" , "archive_write_set_format_pax.c" , "archive_write_set_format_raw.c" , "archive_write_set_format_shar.c" , "archive_write_set_format_ustar.c" , "archive_write_set_format_v7tar.c" , "archive_write_set_format_warc.c" , "archive_write_set_format_xar.c" , "archive_write_set_format_zip.c" , "archive_write_set_options.c" , "archive_write_set_passphrase.c" , "filter_fork_posix.c" , "filter_fork_windows.c" , "xxhash.c" ] } , "libarchive_public_hheaders": { "type": ["@", "rules", "data", "staged"] , "srcs": ["archive.h", "archive_entry.h"] } , "libarchive_private_hheaders": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "archive_acl_private.h" , "archive_blake2.h" , "archive_blake2_impl.h" , "archive_cmdline_private.h" , "archive_crc32.h" , "archive_cryptor_private.h" , "archive_digest_private.h" , "archive_endian.h" , "archive_entry_locale.h" , "archive_entry_private.h" , "archive_getdate.h" , "archive_hmac_private.h" , "archive_openssl_evp_private.h" , "archive_openssl_hmac_private.h" , "archive_options_private.h" , "archive_pack_dev.h" , "archive_pathmatch.h" , "archive_platform.h" , "archive_platform_acl.h" , "archive_platform_xattr.h" , "archive_ppmd7_private.h" , "archive_ppmd8_private.h" , "archive_ppmd_private.h" , "archive_private.h" , "archive_random_private.h" , "archive_rb.h" , "archive_read_disk_private.h" , "archive_read_private.h" , "archive_string.h" , "archive_string_composition.h" , "archive_windows.h" , "archive_write_disk_private.h" , "archive_write_private.h" , "archive_write_set_format_private.h" , "archive_xxhash.h" , "filter_fork.h" ] } , "libarchive": { "type": ["@", "rules", "CC", "library"] , "name": ["libarchive"] , "pure C": ["yes"] , "srcs": ["libarchive_csources"] , "hdrs": ["libarchive_public_hheaders"] , "private-hdrs": ["config.h", "libarchive_public_hheaders", "libarchive_private_hheaders"] , "deps": [ ["src", "archive_acl"] , ["src", "archive_bzip2"] , ["src", "archive_crypt"] , ["src", "archive_crypto"] , ["src", "archive_hidden_symbols"] , ["src", "archive_libb2"] , ["src", "archive_libgcc"] , ["src", "archive_libmd"] , ["src", "archive_libxml2"] , ["src", "archive_lz4"] , ["src", "archive_lzma"] , ["src", "archive_lzo"] , ["src", "archive_lzstd"] , ["src", "archive_mbedtls"] , ["src", "archive_nettle"] , ["src", "archive_pcre"] , ["src", "archive_pcreposix"] , ["src", "archive_pthread"] , ["src", "archive_regex"] , ["src", "archive_richacl"] , ["src", "archive_xattr"] , ["src", "archive_zlib"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/000077500000000000000000000000001516554100600224735ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/TARGETS.archive000066400000000000000000000401201516554100600251440ustar00rootroot00000000000000{ "archive_pthread": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS"] , "name": ["archive_pthread"] , "private-ldflags": { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "OS"}, "$2": "linux"} , "then": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } } , "archive_crypt": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "ENABLE_CNG", "USE_SYSTEM_LIBS"] , "name": ["curl_crypt"] , "private-ldflags": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_CNG"} , {"type": "var", "name": "USE_SYSTEM_LIBS"} ] } , "then": ["-lbcrypt"] } ] ] } , "deps": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_CNG"} , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": false , "else": true } ] } , "then": [["@", "bcrypt", "", "bcrypt"]] } ] ] } } , "archive_zlib": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_ZLIB"] , "name": ["archive_zlib"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZLIB"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lz"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZLIB"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "zlib", "", "zlib"]] } } } , "archive_bzip2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_BZip2"] , "name": ["archive_bzip2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_BZip2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lbz2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_BZip2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "bzip2", "", "libbz2"]] } } } , "archive_lzma": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LZMA"] , "name": ["archive_lzma"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-llzma"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "lzma", "", "lzma"]] } } } , "archive_lzo": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LZO"] , "name": ["archive_lzo"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZO"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-llzo2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZO"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "lzo2", "", "lzo2"]] } } } , "archive_libb2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LIBB2"] , "name": ["archive_libb2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBB2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lb2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBB2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "b2", "", "b2"]] } } } , "archive_lz4": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LZ4"] , "name": ["archive_lz4"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZ4"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-llz4"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZ4"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "lz4", "", "lz4"]] } } } , "archive_lzstd": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_ZSTD"] , "name": ["archive_lzstd"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZSTD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lzstd"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ZSTD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "zstd", "", "zstd"]] } } } , "archive_mbedtls": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_MBEDTLS"] , "name": ["archive_mbedtls"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_MBEDTLS"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lmbedcrypto"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_MBEDTLS"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "mbedtls", "", "mbedcrypto"]] } } } , "archive_nettle": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_NETTLE"] , "name": ["archive_nettle"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_NETTLE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lnettle"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_NETTLE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "nettle", "", "nettle"]] } } } , "archive_libmd": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LIBMD"] , "name": ["archive_libmd"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBMD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lmd"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBMD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "libmd", "", "libmd"]] } } } , "archive_crypto": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_OPENSSL"] , "name": ["archive_crypto"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lcrypto"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ssl", "", "crypto"]] } } } , "archive_libxml2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LIBXML2"] , "name": ["archive_libxml2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBXML2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lxml2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBXML2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "xml2", "", "xml2"]] } } } , "archive_pcreposix": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_PCREPOSIX"] , "name": ["archive_pcreposix"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCREPOSIX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lpcreposix"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCREPOSIX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "pcreposix", "", "pcreposix"]] } } } , "archive_pcre2posix": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_PCRE2POSIX"] , "name": ["archive_pcre2posix"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE2POSIX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lpcre2posix"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE2POSIX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "pcre2posix", "", "pcre2posix"]] } } } , "archive_pcre": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_PCRE"] , "name": ["archive_pcre"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lpcre"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "pcre", "", "pcre"]] } } } , "archive_pcre2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_PCRE2"] , "name": ["archive_pcre2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lpcre2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_PCRE2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "pcre2", "", "pcre2"]] } } } , "archive_regex": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_REGEX"] , "name": ["archive_regex"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_REGEX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lregex"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_REGEX"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "regex", "", "regex"]] } } } , "archive_libgcc": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_LIBGCC"] , "name": ["archive_libgcc"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBGCC"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lgcc"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LIBGCC"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "gcc", "", "gcc"]] } } } , "archive_acl": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_ACL"] , "name": ["archive_acl"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lacl"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "acl", "", "acl"]] } } } , "archive_xattr": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_XATTR", "XATTR_PROVIDER"] , "name": ["archive_xattr"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "XATTR_PROVIDER"} , "case": [["attr", ["-lattr"]]] } } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_XATTR"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "case*" , "expr": {"type": "var", "name": "XATTR_PROVIDER"} , "case": [["attr", [["@", "attr", "", "attr"]]]] } } } } , "archive_richacl": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_RICHACL"] , "name": ["archive_richacl"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_RICHACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lrichacl"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_RICHACL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "richacl", "", "richacl"]] } } } , "archive_hidden_symbols": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["TOOLCHAIN_CONFIG", "HIDE_SYMBOLS"] , "name": ["archive_hidden_symbols"] , "private-ldflags": { "type": "if" , "cond": { "type": "==" , "$1": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "$2": "msvc" } , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "HIDE_SYMBOLS"} , "then": { "type": "case*" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "case": [ ["clang", ["-fvisibility=hidden"]] , ["gnu", ["-fvisibility=hidden"]] , ["intel", ["-fvisibility=hidden"]] ] } } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/TARGETS.curl000066400000000000000000000311561516554100600245010ustar00rootroot00000000000000{ "curl_ares": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "ENABLE_ARES"] , "name": ["curl_ares"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ARES"} , "then": ["-lcares"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ARES"} , "then": [["@", "cares", "", "ares"]] } } } , "curl_ipv6": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "ENABLE_IPV6", "ENABLE_ARES"] , "name": ["curl_ipv6"] , "private-ldflags": { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "ENABLE_IPV6"} , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_ARES"} , "then": false , "else": true } ] } , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "darwin" , [ "-framework" , "SystemConfiguration" , "-framework" , "CoreFoundation" , "-framework" , "CoreServices" ] ] ] } } } , "curl_async_dns": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "ENABLE_THREADED_RESOLVER"] , "name": ["curl_async_dns"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADED_RESOLVER"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "linux" , ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] ] ] } } } , "curl_ssl": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_ENABLE_SSL"] , "name": ["curl_ssl"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": ["-lssl"] , "else": { "type": "fail" , "msg": "libcurl only supported currently with openSSL/boringSSL." } } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} , "then": [["@", "ssl", "", "ssl"], ["@", "ssl", "", "crypto"]] , "else": { "type": "fail" , "msg": "libcurl only supported currently with openSSL/boringSSL." } } } } , "curl_nghttp2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_NGHTTP2"] , "name": ["curl_nghttp2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_NGHTTP2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lnghttp2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_NGHTTP2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "nghttp2", "", "nghttp2"]] } } } , "curl_ngtcp2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_NGTCP2"] , "name": ["curl_ngtcp2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lngtcp2", "-lngthhp3"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ngtcp2", "", "ngtcp2"], ["@", "nghttp3", "", "nghttp3"]] } } } , "curl_quiche": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_QUICHE"] , "name": ["curl_quiche"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lquiche"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "quiche", "", "quiche"]] } } } , "curl_msh3": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_MSH3"] , "name": ["curl_msh3"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_MSH3"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lmsh3"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_MSH3"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "msh3", "", "msh3"]] } } } , "curl_idn2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_LIBIDN2", "OS"] , "name": ["curl_quiche"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_LIBIDN2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["linux", ["-lidn2"]], ["windows", ["-lnormaliz"]]] } } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_LIBIDN2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "USE_NGTCP2"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ ["linux", [["@", "idn2", "", "idn2"]]] , ["windows", [["@", "normaliz", "", "idn2"]]] ] } } } } } , "curl_zlib": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_ZLIB"] , "name": ["curl_zlib"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_ZLIB"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lz"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_ZLIB"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "zlib", "", "zlib"]] } } } , "curl_brotli": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_BROTLI"] , "name": ["curl_brotli"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_BROTLI"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lbrotlidec"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_BROTLI"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "brotli", "", "brotli"]] } } } , "curl_zstd": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_ZSTD"] , "name": ["curl_zstd"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "CURL_ZSTD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-zstd"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "CURL_ZSTD"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "zstd", "", "zstd"]] } } } , "curl_psl": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_USE_LIBPSL"] , "name": ["curl_psl"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBPSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lpsl"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBPSL"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "psl", "", "psl"]] } } } , "curl_ssh2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_USE_LIBSSH2"] , "name": ["curl_ssh2"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lssh"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH2"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ssh2", "", "ssh"]] } } } , "curl_ssh": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_USE_LIBSSH"] , "name": ["curl_ssh"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lssh"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_LIBSSH"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ssh", "", "ssh"]] } } } , "curl_gssapi": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "CURL_USE_GSSAPI"] , "name": ["curl_gssapi"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_GSSAPI"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lgssapi"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "CURL_USE_GSSAPI"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "heimdal", "", "gssapi"]] } } } , "curl_crypt_win32": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "USE_SYSTEM_LIBS"] , "name": ["curl_crypt_win32"] , "private-ldflags": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lbcrypt"] } ] ] } , "deps": { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [ [ "windows" , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "bcrypt", "", "bcrypt"]] } ] ] } } , "curl_hidden_symbols": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["TOOLCHAIN_CONFIG", "CURL_HIDDEN_SYMBOLS"] , "name": ["curl_hidden_symbols"] , "private-ldflags": { "type": "if" , "cond": { "type": "==" , "$1": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "$2": "msvc" } , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "CURL_HIDDEN_SYMBOLS"} , "then": { "type": "case*" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "case": [ ["clang", ["-fvisibility=hidden"]] , ["gnu", ["-fvisibility=hidden"]] , ["sunpro", ["-xldscope=hidden"]] , ["intel", ["-fvisibility=hidden"]] ] } } } } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/TARGETS.git2000066400000000000000000000022451516554100600243760ustar00rootroot00000000000000{ "libgit2": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "ARCH", "TARGET_ARCH"] , "name": ["git2"] , "pkg-name": ["libgit2"] , "pure C": ["yes"] , "private-defines": { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "linux": ["HAVE_QSORT_R_GNU"] , "darwin": ["HAVE_QSORT_R_BSD"] , "bsd": ["HAVE_QSORT_R_BSD"] , "windows": ["HAVE_QSORT_S"] } } , "private-cflags": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": {"arm": ["-Wno-atomic-alignment"]} } , "srcs": [["src/util", "util_sources"], ["src/libgit2", "libgit2_sources"]] , "hdrs": [["include", "git2_public_headers"]] , "private-hdrs": [["src/libgit2", "libgit2_private_headers"]] , "deps": [ ["src/util", "util_compress"] , ["src/util", "util_gssapi"] , ["src/util", "util_http_parser"] , ["src/util", "util_https"] , ["src/util", "util_os"] , ["src/util", "util_regex"] , ["src/util", "util_sha1"] , ["src/util", "util_sha256"] , ["src/util", "util_ssh"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/catch2/000077500000000000000000000000001516554100600236375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/catch2/TARGETS.catch2000066400000000000000000000301701516554100600260370ustar00rootroot00000000000000{ "hdrs": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "benchmark/catch_benchmark.hpp" , "benchmark/catch_benchmark_all.hpp" , "benchmark/catch_chronometer.hpp" , "benchmark/catch_clock.hpp" , "benchmark/catch_constructor.hpp" , "benchmark/catch_environment.hpp" , "benchmark/catch_estimate.hpp" , "benchmark/catch_execution_plan.hpp" , "benchmark/catch_optimizer.hpp" , "benchmark/catch_outlier_classification.hpp" , "benchmark/catch_sample_analysis.hpp" , "benchmark/detail/catch_analyse.hpp" , "benchmark/detail/catch_benchmark_function.hpp" , "benchmark/detail/catch_benchmark_stats.hpp" , "benchmark/detail/catch_benchmark_stats_fwd.hpp" , "benchmark/detail/catch_complete_invoke.hpp" , "benchmark/detail/catch_estimate_clock.hpp" , "benchmark/detail/catch_measure.hpp" , "benchmark/detail/catch_repeat.hpp" , "benchmark/detail/catch_run_for_at_least.hpp" , "benchmark/detail/catch_stats.hpp" , "benchmark/detail/catch_timing.hpp" , "catch_all.hpp" , "catch_approx.hpp" , "catch_assertion_info.hpp" , "catch_assertion_result.hpp" , "catch_config.hpp" , "catch_get_random_seed.hpp" , "catch_message.hpp" , "catch_section_info.hpp" , "catch_session.hpp" , "catch_tag_alias.hpp" , "catch_tag_alias_autoregistrar.hpp" , "catch_template_test_macros.hpp" , "catch_test_case_info.hpp" , "catch_test_macros.hpp" , "catch_test_spec.hpp" , "catch_timer.hpp" , "catch_tostring.hpp" , "catch_totals.hpp" , "catch_translate_exception.hpp" , "catch_version.hpp" , "catch_version_macros.hpp" , "generators/catch_generators.hpp" , "generators/catch_generators_adapters.hpp" , "generators/catch_generators_all.hpp" , "generators/catch_generators_random.hpp" , "generators/catch_generators_range.hpp" , "generators/catch_generator_exception.hpp" , "interfaces/catch_interfaces_all.hpp" , "interfaces/catch_interfaces_capture.hpp" , "interfaces/catch_interfaces_config.hpp" , "interfaces/catch_interfaces_enum_values_registry.hpp" , "interfaces/catch_interfaces_exception.hpp" , "interfaces/catch_interfaces_generatortracker.hpp" , "interfaces/catch_interfaces_registry_hub.hpp" , "interfaces/catch_interfaces_reporter.hpp" , "interfaces/catch_interfaces_reporter_factory.hpp" , "interfaces/catch_interfaces_tag_alias_registry.hpp" , "interfaces/catch_interfaces_testcase.hpp" , "interfaces/catch_interfaces_test_invoker.hpp" , "internal/catch_assertion_handler.hpp" , "internal/catch_case_insensitive_comparisons.hpp" , "internal/catch_case_sensitive.hpp" , "internal/catch_clara.hpp" , "internal/catch_commandline.hpp" , "internal/catch_compare_traits.hpp" , "internal/catch_compiler_capabilities.hpp" , "internal/catch_config_android_logwrite.hpp" , "internal/catch_config_counter.hpp" , "internal/catch_config_prefix_messages.hpp" , "internal/catch_config_static_analysis_support.hpp" , "internal/catch_config_uncaught_exceptions.hpp" , "internal/catch_config_wchar.hpp" , "internal/catch_console_colour.hpp" , "internal/catch_console_width.hpp" , "internal/catch_container_nonmembers.hpp" , "internal/catch_context.hpp" , "internal/catch_debugger.hpp" , "internal/catch_debug_console.hpp" , "internal/catch_decomposer.hpp" , "internal/catch_enforce.hpp" , "internal/catch_enum_values_registry.hpp" , "internal/catch_errno_guard.hpp" , "internal/catch_exception_translator_registry.hpp" , "internal/catch_fatal_condition_handler.hpp" , "internal/catch_floating_point_helpers.hpp" , "internal/catch_getenv.hpp" , "internal/catch_istream.hpp" , "internal/catch_is_permutation.hpp" , "internal/catch_jsonwriter.hpp" , "internal/catch_lazy_expr.hpp" , "internal/catch_leak_detector.hpp" , "internal/catch_list.hpp" , "internal/catch_logical_traits.hpp" , "internal/catch_message_info.hpp" , "internal/catch_meta.hpp" , "internal/catch_move_and_forward.hpp" , "internal/catch_noncopyable.hpp" , "internal/catch_optional.hpp" , "internal/catch_output_redirect.hpp" , "internal/catch_parse_numbers.hpp" , "internal/catch_platform.hpp" , "internal/catch_polyfills.hpp" , "internal/catch_preprocessor.hpp" , "internal/catch_preprocessor_internal_stringify.hpp" , "internal/catch_preprocessor_remove_parens.hpp" , "internal/catch_random_floating_point_helpers.hpp" , "internal/catch_random_integer_helpers.hpp" , "internal/catch_random_number_generator.hpp" , "internal/catch_random_seed_generation.hpp" , "internal/catch_reporter_registry.hpp" , "internal/catch_reporter_spec_parser.hpp" , "internal/catch_result_type.hpp" , "internal/catch_reusable_string_stream.hpp" , "internal/catch_run_context.hpp" , "internal/catch_section.hpp" , "internal/catch_sharding.hpp" , "internal/catch_singletons.hpp" , "internal/catch_source_line_info.hpp" , "internal/catch_startup_exception_registry.hpp" , "internal/catch_stdstreams.hpp" , "internal/catch_stream_end_stop.hpp" , "internal/catch_stringref.hpp" , "internal/catch_string_manip.hpp" , "internal/catch_tag_alias_registry.hpp" , "internal/catch_template_test_registry.hpp" , "internal/catch_test_case_info_hasher.hpp" , "internal/catch_test_case_registry_impl.hpp" , "internal/catch_test_case_tracker.hpp" , "internal/catch_test_failure_exception.hpp" , "internal/catch_test_macro_impl.hpp" , "internal/catch_test_registry.hpp" , "internal/catch_test_run_info.hpp" , "internal/catch_test_spec_parser.hpp" , "internal/catch_textflow.hpp" , "internal/catch_to_string.hpp" , "internal/catch_uncaught_exceptions.hpp" , "internal/catch_uniform_floating_point_distribution.hpp" , "internal/catch_uniform_integer_distribution.hpp" , "internal/catch_unique_name.hpp" , "internal/catch_unique_ptr.hpp" , "internal/catch_void_type.hpp" , "internal/catch_wildcard_pattern.hpp" , "internal/catch_windows_h_proxy.hpp" , "internal/catch_xmlwriter.hpp" , "matchers/catch_matchers.hpp" , "matchers/catch_matchers_all.hpp" , "matchers/catch_matchers_container_properties.hpp" , "matchers/catch_matchers_contains.hpp" , "matchers/catch_matchers_exception.hpp" , "matchers/catch_matchers_floating_point.hpp" , "matchers/catch_matchers_predicate.hpp" , "matchers/catch_matchers_quantifiers.hpp" , "matchers/catch_matchers_range_equals.hpp" , "matchers/catch_matchers_string.hpp" , "matchers/catch_matchers_templated.hpp" , "matchers/catch_matchers_vector.hpp" , "matchers/internal/catch_matchers_impl.hpp" , "reporters/catch_reporters_all.hpp" , "reporters/catch_reporter_automake.hpp" , "reporters/catch_reporter_common_base.hpp" , "reporters/catch_reporter_compact.hpp" , "reporters/catch_reporter_console.hpp" , "reporters/catch_reporter_cumulative_base.hpp" , "reporters/catch_reporter_event_listener.hpp" , "reporters/catch_reporter_helpers.hpp" , "reporters/catch_reporter_json.hpp" , "reporters/catch_reporter_junit.hpp" , "reporters/catch_reporter_multi.hpp" , "reporters/catch_reporter_registrars.hpp" , "reporters/catch_reporter_sonarqube.hpp" , "reporters/catch_reporter_streaming_base.hpp" , "reporters/catch_reporter_tap.hpp" , "reporters/catch_reporter_teamcity.hpp" , "reporters/catch_reporter_xml.hpp" ] , "stage": ["catch2"] } , "srcs": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "benchmark/catch_chronometer.cpp" , "benchmark/detail/catch_analyse.cpp" , "benchmark/detail/catch_benchmark_function.cpp" , "benchmark/detail/catch_run_for_at_least.cpp" , "benchmark/detail/catch_stats.cpp" , "catch_approx.cpp" , "catch_assertion_result.cpp" , "catch_config.cpp" , "catch_get_random_seed.cpp" , "catch_message.cpp" , "catch_registry_hub.cpp" , "catch_session.cpp" , "catch_tag_alias_autoregistrar.cpp" , "catch_test_case_info.cpp" , "catch_test_spec.cpp" , "catch_timer.cpp" , "catch_tostring.cpp" , "catch_totals.cpp" , "catch_translate_exception.cpp" , "catch_version.cpp" , "generators/catch_generators.cpp" , "generators/catch_generators_random.cpp" , "generators/catch_generator_exception.cpp" , "interfaces/catch_interfaces_capture.cpp" , "interfaces/catch_interfaces_config.cpp" , "interfaces/catch_interfaces_exception.cpp" , "interfaces/catch_interfaces_generatortracker.cpp" , "interfaces/catch_interfaces_registry_hub.cpp" , "interfaces/catch_interfaces_reporter.cpp" , "interfaces/catch_interfaces_reporter_factory.cpp" , "interfaces/catch_interfaces_testcase.cpp" , "internal/catch_assertion_handler.cpp" , "internal/catch_case_insensitive_comparisons.cpp" , "internal/catch_clara.cpp" , "internal/catch_commandline.cpp" , "internal/catch_console_colour.cpp" , "internal/catch_context.cpp" , "internal/catch_debugger.cpp" , "internal/catch_debug_console.cpp" , "internal/catch_decomposer.cpp" , "internal/catch_enforce.cpp" , "internal/catch_enum_values_registry.cpp" , "internal/catch_errno_guard.cpp" , "internal/catch_exception_translator_registry.cpp" , "internal/catch_fatal_condition_handler.cpp" , "internal/catch_floating_point_helpers.cpp" , "internal/catch_getenv.cpp" , "internal/catch_istream.cpp" , "internal/catch_jsonwriter.cpp" , "internal/catch_lazy_expr.cpp" , "internal/catch_leak_detector.cpp" , "internal/catch_list.cpp" , "internal/catch_message_info.cpp" , "internal/catch_output_redirect.cpp" , "internal/catch_parse_numbers.cpp" , "internal/catch_polyfills.cpp" , "internal/catch_random_number_generator.cpp" , "internal/catch_random_seed_generation.cpp" , "internal/catch_reporter_registry.cpp" , "internal/catch_reporter_spec_parser.cpp" , "internal/catch_result_type.cpp" , "internal/catch_reusable_string_stream.cpp" , "internal/catch_run_context.cpp" , "internal/catch_section.cpp" , "internal/catch_singletons.cpp" , "internal/catch_source_line_info.cpp" , "internal/catch_startup_exception_registry.cpp" , "internal/catch_stdstreams.cpp" , "internal/catch_stringref.cpp" , "internal/catch_string_manip.cpp" , "internal/catch_tag_alias_registry.cpp" , "internal/catch_test_case_info_hasher.cpp" , "internal/catch_test_case_registry_impl.cpp" , "internal/catch_test_case_tracker.cpp" , "internal/catch_test_failure_exception.cpp" , "internal/catch_test_registry.cpp" , "internal/catch_test_spec_parser.cpp" , "internal/catch_textflow.cpp" , "internal/catch_uncaught_exceptions.cpp" , "internal/catch_wildcard_pattern.cpp" , "internal/catch_xmlwriter.cpp" , "matchers/catch_matchers.cpp" , "matchers/catch_matchers_container_properties.cpp" , "matchers/catch_matchers_exception.cpp" , "matchers/catch_matchers_floating_point.cpp" , "matchers/catch_matchers_predicate.cpp" , "matchers/catch_matchers_quantifiers.cpp" , "matchers/catch_matchers_string.cpp" , "matchers/catch_matchers_templated.cpp" , "matchers/internal/catch_matchers_impl.cpp" , "reporters/catch_reporter_automake.cpp" , "reporters/catch_reporter_common_base.cpp" , "reporters/catch_reporter_compact.cpp" , "reporters/catch_reporter_console.cpp" , "reporters/catch_reporter_cumulative_base.cpp" , "reporters/catch_reporter_event_listener.cpp" , "reporters/catch_reporter_helpers.cpp" , "reporters/catch_reporter_json.cpp" , "reporters/catch_reporter_junit.cpp" , "reporters/catch_reporter_multi.cpp" , "reporters/catch_reporter_registrars.cpp" , "reporters/catch_reporter_sonarqube.cpp" , "reporters/catch_reporter_streaming_base.cpp" , "reporters/catch_reporter_tap.cpp" , "reporters/catch_reporter_teamcity.cpp" , "reporters/catch_reporter_xml.cpp" ] , "stage": ["catch2"] } , "main": { "type": ["@", "rules", "data", "staged"] , "srcs": ["internal/catch_main.cpp"] , "stage": ["catch2"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/common/000077500000000000000000000000001516554100600237635ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/common/TARGETS.lzma000066400000000000000000000007561516554100600257710ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ENABLE_THREADS"] , "hdrs": { "type": "++" , "$1": [ [ "mythread.h" , "sysdefs.h" , "tuklib_common.h" , "tuklib_config.h" , "tuklib_integer.h" , "tuklib_physmem.c" , "tuklib_physmem.h" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["tuklib_cpucores.h"] } ] } } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/compiler/000077500000000000000000000000001516554100600243055ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/compiler/TARGETS.grpc000066400000000000000000000035031516554100600262740ustar00rootroot00000000000000{ "proto_parser_helper": { "type": ["@", "rules", "CC", "library"] , "name": ["proto_parser_helper"] , "stage": ["src", "compiler"] , "srcs": ["proto_parser_helper.cc"] , "hdrs": ["proto_parser_helper.h"] } , "grpc_plugin_support": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_plugin_support"] , "stage": ["src", "compiler"] , "srcs": [ "cpp_generator.cc" , "csharp_generator.cc" , "node_generator.cc" , "objective_c_generator.cc" , "php_generator.cc" , "python_generator.cc" , "ruby_generator.cc" ] , "hdrs": [ "config.h" , "config_protobuf.h" , "cpp_generator.h" , "cpp_generator_helpers.h" , "cpp_plugin.h" , "csharp_generator.h" , "csharp_generator_helpers.h" , "generator_helpers.h" , "node_generator.h" , "node_generator_helpers.h" , "objective_c_generator.h" , "objective_c_generator_helpers.h" , "php_generator.h" , "php_generator_helpers.h" , "protobuf_plugin.h" , "python_generator.h" , "python_generator_helpers.h" , "python_private_generator.h" , "ruby_generator.h" , "ruby_generator_helpers-inl.h" , "ruby_generator_map-inl.h" , "ruby_generator_string-inl.h" , "schema_interface.h" ] , "deps": [ "proto_parser_helper" , ["@", "protobuf", "", "libprotobuf"] , ["@", "protobuf", "", "libprotoc"] , ["", "grpc++_config_proto"] ] } , "grpc_cpp_plugin": { "type": "configure" , "target": "grpc_cpp_plugin (unconfigured)" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "grpc_cpp_plugin (unconfigured)": { "type": ["@", "rules", "CC", "binary"] , "name": ["grpc_cpp_plugin"] , "srcs": ["cpp_plugin.cc"] , "private-deps": ["grpc_plugin_support"] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/000077500000000000000000000000001516554100600234235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/TARGETS.grpc000066400000000000000000006703671516554100600254340ustar00rootroot00000000000000{ "channel_fwd": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_fwd"] , "stage": ["src", "core"] , "hdrs": ["lib/channel/channel_fwd.h"] } , "dump_args": { "type": ["@", "rules", "CC", "library"] , "name": ["dump_args"] , "stage": ["src", "core"] , "srcs": ["util/dump_args.cc"] , "hdrs": ["util/dump_args.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] ] } , "slice_cast": { "type": ["@", "rules", "CC", "library"] , "name": ["slice_cast"] , "stage": ["src", "core"] , "hdrs": [["include/grpc", "slice_cast_headers"]] } , "event_engine_extensions": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_extensions"] , "stage": ["src", "core"] , "hdrs": [ "lib/event_engine/extensions/can_track_errors.h" , "lib/event_engine/extensions/chaotic_good_extension.h" , "lib/event_engine/extensions/supports_fd.h" , "lib/event_engine/extensions/tcp_trace.h" ] , "deps": [ "memory_quota" , "slice_buffer" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] , ["", "tcp_tracer"] ] } , "event_engine_common": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_common"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/event_engine.cc" , "lib/event_engine/resolved_address.cc" , "lib/event_engine/slice.cc" , "lib/event_engine/slice_buffer.cc" ] , "hdrs": [ "lib/event_engine/extensions/can_track_errors.h" , "lib/event_engine/handle_containers.h" , "lib/event_engine/resolved_address_internal.h" , ["include/grpc", "slice_buffer_headers"] ] , "deps": [ "resolved_address" , "slice" , "slice_buffer" , "slice_cast" , "slice_refcount" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/utility", "utility"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "transport_framing_endpoint_extension": { "type": ["@", "rules", "CC", "library"] , "name": ["transport_framing_endpoint_extension"] , "stage": ["src", "core"] , "hdrs": ["lib/transport/transport_framing_endpoint_extension.h"] , "deps": [ "slice_buffer" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "latent_see": { "type": ["@", "rules", "CC", "library"] , "name": ["latent_see"] , "stage": ["src", "core"] , "srcs": ["util/latent_see.cc"] , "hdrs": ["util/latent_see.h"] , "deps": [ "per_cpu" , "ring_buffer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] ] } , "ring_buffer": { "type": ["@", "rules", "CC", "library"] , "name": ["ring_buffer"] , "stage": ["src", "core"] , "hdrs": ["util/ring_buffer.h"] , "deps": [["@", "absl", "absl/types", "optional"], ["", "gpr_platform"]] } , "transport_fwd": { "type": ["@", "rules", "CC", "library"] , "name": ["transport_fwd"] , "stage": ["src", "core"] , "hdrs": ["lib/transport/transport_fwd.h"] } , "server_call_tracer_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["server_call_tracer_filter"] , "stage": ["src", "core"] , "srcs": ["server/server_call_tracer_filter.cc"] , "hdrs": ["server/server_call_tracer_filter.h"] , "deps": [ "arena_promise" , "call_finalization" , "cancel_callback" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "latent_see" , "map" , "pipe" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "call_tracer"] , ["", "config"] , ["", "gpr_platform"] , ["", "grpc_base"] ] } , "atomic_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["atomic_utils"] , "stage": ["src", "core"] , "hdrs": ["util/atomic_utils.h"] , "deps": [["", "gpr"]] } , "metadata_compression_traits": { "type": ["@", "rules", "CC", "library"] , "name": ["metadata_compression_traits"] , "stage": ["src", "core"] , "hdrs": ["lib/transport/metadata_compression_traits.h"] , "deps": [["", "gpr_platform"]] } , "metadata_info": { "type": ["@", "rules", "CC", "library"] , "name": ["metadata_info"] , "stage": ["src", "core"] , "srcs": ["lib/transport/metadata_info.cc"] , "hdrs": ["lib/transport/metadata_info.h"] , "deps": [ "channel_args" , "hpack_constants" , "metadata_batch" , "slice" , ["", "call_tracer"] , ["", "gpr_platform"] , ["", "grpc_base"] ] } , "experiments": { "type": ["@", "rules", "CC", "library"] , "name": ["experiments"] , "stage": ["src", "core"] , "srcs": ["lib/experiments/config.cc", "lib/experiments/experiments.cc"] , "hdrs": ["lib/experiments/config.h", "lib/experiments/experiments.h"] , "deps": [ "no_destruct" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "config_vars"] , ["", "gpr"] ] } , "init_internally": { "type": ["@", "rules", "CC", "library"] , "name": ["init_internally"] , "stage": ["src", "core"] , "srcs": ["lib/surface/init_internally.cc"] , "hdrs": ["lib/surface/init_internally.h"] , "deps": [["", "gpr_platform"]] } , "useful": { "type": ["@", "rules", "CC", "library"] , "name": ["useful"] , "stage": ["src", "core"] , "hdrs": ["util/useful.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/numeric", "bits"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr_platform"] ] } , "unique_ptr_with_bitset": { "type": ["@", "rules", "CC", "library"] , "name": ["unique_ptr_with_bitset"] , "stage": ["src", "core"] , "hdrs": ["util/unique_ptr_with_bitset.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/numeric", "bits"] , ["", "gpr_platform"] ] } , "examine_stack": { "type": ["@", "rules", "CC", "library"] , "name": ["examine_stack"] , "stage": ["src", "core"] , "srcs": ["util/examine_stack.cc"] , "hdrs": ["util/examine_stack.h"] , "deps": [["@", "absl", "absl/types", "optional"], ["", "gpr_platform"]] } , "gpr_atm": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr_atm"] , "stage": ["src", "core"] , "hdrs": [["include/grpc", "gpr_atm_headers"]] , "deps": ["useful", ["", "gpr_platform"]] } , "gpr_manual_constructor": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr_manual_constructor"] , "stage": ["src", "core"] , "hdrs": ["util/manual_constructor.h"] , "deps": ["construct_destruct", ["", "gpr_platform"]] } , "gpr_spinlock": { "type": ["@", "rules", "CC", "library"] , "name": ["gpr_spinlock"] , "stage": ["src", "core"] , "hdrs": ["util/spinlock.h"] , "deps": ["gpr_atm", ["", "gpr_platform"]] } , "env": { "type": ["@", "rules", "CC", "library"] , "name": ["env"] , "stage": ["src", "core"] , "srcs": ["util/linux/env.cc", "util/posix/env.cc", "util/windows/env.cc"] , "hdrs": ["util/env.h"] , "deps": ["tchar", ["@", "absl", "absl/types", "optional"], ["", "gpr_platform"]] } , "directory_reader": { "type": ["@", "rules", "CC", "library"] , "name": ["directory_reader"] , "stage": ["src", "core"] , "srcs": ["util/posix/directory_reader.cc", "util/windows/directory_reader.cc"] , "hdrs": ["util/directory_reader.h"] , "deps": [ ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "chunked_vector": { "type": ["@", "rules", "CC", "library"] , "name": ["chunked_vector"] , "stage": ["src", "core"] , "hdrs": ["util/chunked_vector.h"] , "deps": [ "arena" , "gpr_manual_constructor" , ["@", "absl", "absl/log", "check"] , ["", "gpr"] ] } , "construct_destruct": { "type": ["@", "rules", "CC", "library"] , "name": ["construct_destruct"] , "stage": ["src", "core"] , "hdrs": ["util/construct_destruct.h"] , "deps": [["", "gpr_platform"]] } , "rpc_status_grpc_internal_default_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["rpc_status_grpc_internal_default_proto"] , "srcs": [["@", "google_apis", "", "google/rpc/status.proto"]] } , "status_helper_private_proto_deps": { "type": ["@", "rules", "CC", "library"] , "name": ["status_helper_private_proto_deps"] , "proto": ["rpc_status_grpc_internal_default_proto"] } , "status_helper": { "type": ["@", "rules", "CC", "library"] , "name": ["status_helper"] , "stage": ["src", "core"] , "srcs": ["util/status_helper.cc"] , "hdrs": ["util/status_helper.h"] , "deps": [ "percent_encoding" , "slice" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["", "debug_location"] , ["", "gpr"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] , "private-deps": ["status_helper_private_proto_deps"] } , "unique_type_name": { "type": ["@", "rules", "CC", "library"] , "name": ["unique_type_name"] , "stage": ["src", "core"] , "hdrs": ["util/unique_type_name.h"] , "deps": ["useful", ["@", "absl", "absl/strings", "strings"], ["", "gpr_platform"]] } , "validation_errors": { "type": ["@", "rules", "CC", "library"] , "name": ["validation_errors"] , "stage": ["src", "core"] , "srcs": ["util/validation_errors.cc"] , "hdrs": ["util/validation_errors.h"] , "deps": [ ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "overload": { "type": ["@", "rules", "CC", "library"] , "name": ["overload"] , "stage": ["src", "core"] , "hdrs": ["util/overload.h"] , "deps": [["", "gpr_platform"]] } , "match": { "type": ["@", "rules", "CC", "library"] , "name": ["match"] , "stage": ["src", "core"] , "hdrs": ["util/match.h"] , "deps": ["overload", ["@", "absl", "absl/types", "variant"], ["", "gpr_platform"]] } , "table": { "type": ["@", "rules", "CC", "library"] , "name": ["table"] , "stage": ["src", "core"] , "hdrs": ["util/table.h"] , "deps": [ "bitset" , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/utility", "utility"] , ["", "gpr_platform"] ] } , "packed_table": { "type": ["@", "rules", "CC", "library"] , "name": ["packed_table"] , "stage": ["src", "core"] , "hdrs": ["util/packed_table.h"] , "deps": ["sorted_pack", "table", ["", "gpr_platform"]] } , "bitset": { "type": ["@", "rules", "CC", "library"] , "name": ["bitset"] , "stage": ["src", "core"] , "hdrs": ["util/bitset.h"] , "deps": ["useful", ["", "gpr_platform"]] } , "no_destruct": { "type": ["@", "rules", "CC", "library"] , "name": ["no_destruct"] , "stage": ["src", "core"] , "hdrs": ["util/no_destruct.h"] , "deps": ["construct_destruct", ["", "gpr_platform"]] } , "tchar": { "type": ["@", "rules", "CC", "library"] , "name": ["tchar"] , "stage": ["src", "core"] , "srcs": ["util/tchar.cc"] , "hdrs": ["util/tchar.h"] , "deps": [["", "gpr_platform"]] } , "poll": { "type": ["@", "rules", "CC", "library"] , "name": ["poll"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/poll.h"] , "deps": [ "construct_destruct" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "status_flag": { "type": ["@", "rules", "CC", "library"] , "name": ["status_flag"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/status_flag.h"] , "deps": [ "promise_status" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "map_pipe": { "type": ["@", "rules", "CC", "library"] , "name": ["map_pipe"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/map_pipe.h"] , "deps": [ "for_each" , "map" , "pipe" , "poll" , "promise_factory" , "try_seq" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "1999": { "type": ["@", "rules", "CC", "library"] , "name": ["1999"] , "stage": ["src", "core"] , "srcs": ["lib/promise/party.cc"] , "hdrs": ["lib/promise/party.h"] , "deps": [ "activity" , "arena" , "construct_destruct" , "context" , "event_engine_context" , "latent_see" , "poll" , "promise_factory" , "ref_counted" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] ] } , "context": { "type": ["@", "rules", "CC", "library"] , "name": ["context"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/context.h"] , "deps": [ "down_cast" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/meta", "type_traits"] , ["", "gpr"] ] } , "map": { "type": ["@", "rules", "CC", "library"] , "name": ["map"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/map.h"] , "deps": [ "poll" , "promise_like" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "promise_variant": { "type": ["@", "rules", "CC", "library"] , "name": ["promise_variant"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/promise_variant.h"] , "deps": [["@", "absl", "absl/types", "variant"]] } , "match_promise": { "type": ["@", "rules", "CC", "library"] , "name": ["match_promise"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/match_promise.h"] , "deps": [ "overload" , "poll" , "promise_factory" , "promise_like" , "promise_variant" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr_platform"] ] } , "sleep": { "type": ["@", "rules", "CC", "library"] , "name": ["sleep"] , "stage": ["src", "core"] , "srcs": ["lib/promise/sleep.cc"] , "hdrs": ["lib/promise/sleep.h"] , "deps": [ "activity" , "context" , "event_engine_context" , "poll" , "time" , ["@", "absl", "absl/status", "status"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] ] } , "wait_for_callback": { "type": ["@", "rules", "CC", "library"] , "name": ["wait_for_callback"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/wait_for_callback.h"] , "deps": [ "activity" , "poll" , ["@", "absl", "absl/base", "core_headers"] , ["", "gpr"] ] } , "arena_promise": { "type": ["@", "rules", "CC", "library"] , "name": ["arena_promise"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/arena_promise.h"] , "deps": [ "arena" , "construct_destruct" , "context" , "poll" , ["@", "absl", "absl/meta", "type_traits"] , ["", "gpr_platform"] ] } , "promise_like": { "type": ["@", "rules", "CC", "library"] , "name": ["promise_like"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/promise_like.h"] , "deps": [ "poll" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/meta", "type_traits"] , ["", "gpr_platform"] ] } , "cancel_callback": { "type": ["@", "rules", "CC", "library"] , "name": ["cancel_callback"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/cancel_callback.h"] , "deps": ["arena", "context", "promise_like", ["", "gpr_platform"]] } , "promise_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["promise_factory"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/promise_factory.h"] , "deps": [ "promise_like" , ["@", "absl", "absl/meta", "type_traits"] , ["", "gpr_platform"] ] } , "if": { "type": ["@", "rules", "CC", "library"] , "name": ["if"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/if.h"] , "deps": [ "construct_destruct" , "poll" , "promise_factory" , "promise_like" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr_platform"] ] } , "switch": { "type": ["@", "rules", "CC", "library"] , "name": ["switch"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/switch.h"] , "deps": [ "if" , "promise_factory" , "promise_variant" , ["", "gpr"] , ["", "gpr_platform"] ] } , "promise_status": { "type": ["@", "rules", "CC", "library"] , "name": ["promise_status"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/status.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "gpr_platform"] ] } , "race": { "type": ["@", "rules", "CC", "library"] , "name": ["race"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/race.h"] , "deps": [["", "gpr_platform"]] } , "prioritized_race": { "type": ["@", "rules", "CC", "library"] , "name": ["prioritized_race"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/prioritized_race.h"] , "deps": [["", "gpr_platform"]] } , "loop": { "type": ["@", "rules", "CC", "library"] , "name": ["loop"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/loop.h"] , "deps": [ "construct_destruct" , "poll" , "promise_factory" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "join_state": { "type": ["@", "rules", "CC", "library"] , "name": ["join_state"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/join_state.h"] , "deps": [ "bitset" , "construct_destruct" , "poll" , "promise_like" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "join": { "type": ["@", "rules", "CC", "library"] , "name": ["join"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/join.h"] , "deps": [ "join_state" , "map" , "promise_factory" , ["@", "absl", "absl/meta", "type_traits"] , ["", "gpr_platform"] ] } , "try_join": { "type": ["@", "rules", "CC", "library"] , "name": ["try_join"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/try_join.h"] , "deps": [ "join_state" , "map" , "poll" , "status_flag" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "gpr_platform"] ] } , "all_ok": { "type": ["@", "rules", "CC", "library"] , "name": ["all_ok"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/all_ok.h"] , "deps": [ "join_state" , "map" , "poll" , "promise_factory" , "status_flag" , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "gpr_platform"] ] } , "basic_seq": { "type": ["@", "rules", "CC", "library"] , "name": ["basic_seq"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/basic_seq.h"] , "deps": ["construct_destruct", "poll", "promise_factory", ["", "gpr_platform"]] } , "seq_state": { "type": ["@", "rules", "CC", "library"] , "name": ["seq_state"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/seq_state.h"] , "deps": [ "construct_destruct" , "poll" , "promise_factory" , "promise_like" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "debug_location"] , ["", "grpc_trace"] ] } , "seq": { "type": ["@", "rules", "CC", "library"] , "name": ["seq"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/seq.h"] , "deps": [ "basic_seq" , "poll" , "promise_like" , "seq_state" , ["@", "absl", "absl/log", "log"] , ["", "debug_location"] , ["", "gpr_platform"] ] } , "try_seq": { "type": ["@", "rules", "CC", "library"] , "name": ["try_seq"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/try_seq.h"] , "deps": [ "basic_seq" , "poll" , "promise_like" , "promise_status" , "seq_state" , "status_flag" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "gpr_platform"] ] } , "activity": { "type": ["@", "rules", "CC", "library"] , "name": ["activity"] , "stage": ["src", "core"] , "srcs": ["lib/promise/activity.cc"] , "hdrs": ["lib/promise/activity.h"] , "deps": [ "atomic_utils" , "construct_destruct" , "context" , "dump_args" , "latent_see" , "no_destruct" , "poll" , "promise_factory" , "promise_status" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_trace"] , ["", "orphanable"] ] } , "exec_ctx_wakeup_scheduler": { "type": ["@", "rules", "CC", "library"] , "name": ["exec_ctx_wakeup_scheduler"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/exec_ctx_wakeup_scheduler.h"] , "deps": [ "closure" , "error" , ["@", "absl", "absl/status", "status"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr_platform"] ] } , "event_engine_wakeup_scheduler": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_wakeup_scheduler"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/event_engine_wakeup_scheduler.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr_platform"] ] } , "wait_set": { "type": ["@", "rules", "CC", "library"] , "name": ["wait_set"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/wait_set.h"] , "deps": [ "activity" , "poll" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/hash", "hash"] , ["", "gpr_platform"] ] } , "latch": { "type": ["@", "rules", "CC", "library"] , "name": ["latch"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/latch.h"] , "deps": [ "activity" , "poll" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "inter_activity_latch": { "type": ["@", "rules", "CC", "library"] , "name": ["inter_activity_latch"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/inter_activity_latch.h"] , "deps": [ "activity" , "poll" , "wait_set" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "interceptor_list": { "type": ["@", "rules", "CC", "library"] , "name": ["interceptor_list"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/interceptor_list.h"] , "deps": [ "arena" , "construct_destruct" , "context" , "poll" , "promise_factory" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "debug_location"] , ["", "gpr"] ] } , "pipe": { "type": ["@", "rules", "CC", "library"] , "name": ["pipe"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/pipe.h"] , "deps": [ "activity" , "arena" , "context" , "if" , "interceptor_list" , "map" , "poll" , "seq" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "debug_location"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "promise_mutex": { "type": ["@", "rules", "CC", "library"] , "name": ["promise_mutex"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/promise_mutex.h"] , "deps": ["activity", "poll", ["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "inter_activity_pipe": { "type": ["@", "rules", "CC", "library"] , "name": ["inter_activity_pipe"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/inter_activity_pipe.h"] , "deps": [ "activity" , "poll" , "ref_counted" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/types", "optional"] , ["", "debug_location"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "mpsc": { "type": ["@", "rules", "CC", "library"] , "name": ["mpsc"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/mpsc.h"] , "deps": [ "activity" , "dump_args" , "poll" , "ref_counted" , "status_flag" , "wait_set" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "observable": { "type": ["@", "rules", "CC", "library"] , "name": ["observable"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/observable.h"] , "deps": [ "activity" , "poll" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["", "gpr"] ] } , "for_each": { "type": ["@", "rules", "CC", "library"] , "name": ["for_each"] , "stage": ["src", "core"] , "hdrs": ["lib/promise/for_each.h"] , "deps": [ "activity" , "construct_destruct" , "poll" , "promise_factory" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "ref_counted": { "type": ["@", "rules", "CC", "library"] , "name": ["ref_counted"] , "stage": ["src", "core"] , "hdrs": ["util/ref_counted.h"] , "deps": [ "atomic_utils" , "down_cast" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["", "debug_location"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "dual_ref_counted": { "type": ["@", "rules", "CC", "library"] , "name": ["dual_ref_counted"] , "stage": ["src", "core"] , "hdrs": ["util/dual_ref_counted.h"] , "deps": [ ["", "debug_location"] , ["", "gpr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "ref_counted_string": { "type": ["@", "rules", "CC", "library"] , "name": ["ref_counted_string"] , "stage": ["src", "core"] , "srcs": ["util/ref_counted_string.cc"] , "hdrs": ["util/ref_counted_string.h"] , "deps": [ "ref_counted" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "uuid_v4": { "type": ["@", "rules", "CC", "library"] , "name": ["uuid_v4"] , "stage": ["src", "core"] , "srcs": ["util/uuid_v4.cc"] , "hdrs": ["util/uuid_v4.h"] , "deps": [["@", "absl", "absl/strings", "str_format"], ["", "gpr"]] } , "handshaker_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["handshaker_factory"] , "stage": ["src", "core"] , "hdrs": ["handshaker/handshaker_factory.h"] , "deps": ["channel_args", "iomgr_fwd", ["", "gpr_platform"]] } , "handshaker_registry": { "type": ["@", "rules", "CC", "library"] , "name": ["handshaker_registry"] , "stage": ["src", "core"] , "srcs": ["handshaker/handshaker_registry.cc"] , "hdrs": ["handshaker/handshaker_registry.h"] , "deps": ["channel_args", "handshaker_factory", "iomgr_fwd", ["", "gpr_platform"]] } , "tcp_connect_handshaker": { "type": ["@", "rules", "CC", "library"] , "name": ["tcp_connect_handshaker"] , "stage": ["src", "core"] , "srcs": ["handshaker/tcp_connect/tcp_connect_handshaker.cc"] , "hdrs": ["handshaker/tcp_connect/tcp_connect_handshaker.h"] , "deps": [ "channel_args" , "channel_args_endpoint_config" , "closure" , "error" , "handshaker_factory" , "handshaker_registry" , "iomgr_fwd" , "pollset_set" , "resolved_address" , "slice" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "endpoint_info_handshaker": { "type": ["@", "rules", "CC", "library"] , "name": ["endpoint_info_handshaker"] , "stage": ["src", "core"] , "srcs": ["handshaker/endpoint_info/endpoint_info_handshaker.cc"] , "hdrs": ["handshaker/endpoint_info/endpoint_info_handshaker.h"] , "deps": [ "channel_args" , "closure" , "handshaker_factory" , "handshaker_registry" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "handshaker"] , ["", "iomgr"] , ["", "ref_counted_ptr"] ] } , "channel_creds_registry": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_creds_registry"] , "stage": ["src", "core"] , "hdrs": ["lib/security/credentials/channel_creds_registry.h"] , "deps": [ "json" , "json_args" , "ref_counted" , "validation_errors" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "event_engine_memory_allocator": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_memory_allocator"] , "stage": ["src", "core"] , "hdrs": [["include/grpc", "event_engine_memory_allocator_headers"]] , "deps": ["slice", ["@", "absl", "absl/strings", "strings"], ["", "gpr_platform"]] } , "event_engine_memory_allocator_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_memory_allocator_factory"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/memory_allocator_factory.h"] , "deps": [ "event_engine_memory_allocator" , "memory_quota" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "memory_quota": { "type": ["@", "rules", "CC", "library"] , "name": ["memory_quota"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/memory_quota.cc"] , "hdrs": ["lib/resource_quota/memory_quota.h"] , "deps": [ "activity" , "event_engine_memory_allocator" , "exec_ctx_wakeup_scheduler" , "experiments" , "loop" , "map" , "periodic_update" , "poll" , "race" , "seq" , "slice_refcount" , "time" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "periodic_update": { "type": ["@", "rules", "CC", "library"] , "name": ["periodic_update"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/periodic_update.cc"] , "hdrs": ["lib/resource_quota/periodic_update.h"] , "deps": [ "time" , "useful" , ["@", "absl", "absl/functional", "function_ref"] , ["", "gpr_platform"] ] } , "arena": { "type": ["@", "rules", "CC", "library"] , "name": ["arena"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/arena.cc"] , "hdrs": ["lib/resource_quota/arena.h"] , "deps": [ "construct_destruct" , "context" , "event_engine_memory_allocator" , "memory_quota" , "resource_quota" , ["@", "absl", "absl/log", "log"] , ["", "gpr"] ] } , "thread_quota": { "type": ["@", "rules", "CC", "library"] , "name": ["thread_quota"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/thread_quota.cc"] , "hdrs": ["lib/resource_quota/thread_quota.h"] , "deps": [ "ref_counted" , ["@", "absl", "absl/base", "core_headers"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "connection_quota": { "type": ["@", "rules", "CC", "library"] , "name": ["connection_quota"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/connection_quota.cc"] , "hdrs": ["lib/resource_quota/connection_quota.h"] , "deps": [ "memory_quota" , "ref_counted" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "resource_quota": { "type": ["@", "rules", "CC", "library"] , "name": ["resource_quota"] , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/resource_quota.cc"] , "hdrs": ["lib/resource_quota/resource_quota.h"] , "deps": [ "connection_quota" , "memory_quota" , "ref_counted" , "thread_quota" , "useful" , ["@", "absl", "absl/strings", "strings"] , ["", "channel_arg_names"] , ["", "cpp_impl_of"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "request_buffer": { "type": ["@", "rules", "CC", "library"] , "name": ["request_buffer"] , "stage": ["src", "core"] , "srcs": ["call/request_buffer.cc"] , "hdrs": ["call/request_buffer.h"] , "deps": [ "call_spine" , "match" , "message" , "metadata" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "slice_refcount": { "type": ["@", "rules", "CC", "library"] , "name": ["slice_refcount"] , "stage": ["src", "core"] , "hdrs": ["lib/slice/slice_refcount.h", ["include/grpc", "slice.h"]] , "deps": [ ["@", "absl", "absl/log", "log"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "slice": { "type": ["@", "rules", "CC", "library"] , "name": ["slice"] , "stage": ["src", "core"] , "srcs": ["lib/slice/slice.cc", "lib/slice/slice_string_helpers.cc"] , "hdrs": [ "lib/slice/slice.h" , "lib/slice/slice_internal.h" , "lib/slice/slice_string_helpers.h" , ["include/grpc", "slice.h"] ] , "deps": [ "slice_cast" , "slice_refcount" , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "slice_buffer": { "type": ["@", "rules", "CC", "library"] , "name": ["slice_buffer"] , "stage": ["src", "core"] , "srcs": ["lib/slice/slice_buffer.cc"] , "hdrs": ["lib/slice/slice_buffer.h", ["include/grpc", "slice_buffer.h"]] , "deps": [ "slice" , "slice_refcount" , ["@", "absl", "absl/log", "check"] , ["", "gpr"] ] } , "error": { "type": ["@", "rules", "CC", "library"] , "name": ["error"] , "stage": ["src", "core"] , "srcs": ["lib/iomgr/error.cc"] , "hdrs": ["lib/iomgr/error.h"] , "deps": [ "gpr_spinlock" , "slice" , "slice_refcount" , "status_helper" , "strerror" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] ] } , "closure": { "type": ["@", "rules", "CC", "library"] , "name": ["closure"] , "stage": ["src", "core"] , "srcs": ["lib/iomgr/closure.cc"] , "hdrs": ["lib/iomgr/closure.h"] , "deps": [ "error" , "gpr_manual_constructor" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["", "debug_location"] , ["", "gpr"] ] } , "time": { "type": ["@", "rules", "CC", "library"] , "name": ["time"] , "stage": ["src", "core"] , "srcs": ["util/time.cc"] , "hdrs": ["util/time.h"] , "deps": [ "no_destruct" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "iomgr_port": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr_port"] , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/port.h"] , "deps": [["", "gpr_platform"]] } , "iomgr_fwd": { "type": ["@", "rules", "CC", "library"] , "name": ["iomgr_fwd"] , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/iomgr_fwd.h"] , "deps": [["", "gpr_platform"]] } , "grpc_sockaddr": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_sockaddr"] , "stage": ["src", "core"] , "srcs": ["lib/iomgr/sockaddr_utils_posix.cc", "lib/iomgr/socket_utils_windows.cc"] , "hdrs": [ "lib/iomgr/sockaddr.h" , "lib/iomgr/sockaddr_posix.h" , "lib/iomgr/sockaddr_windows.h" , "lib/iomgr/socket_utils.h" ] , "deps": ["iomgr_port", ["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "avl": { "type": ["@", "rules", "CC", "library"] , "name": ["avl"] , "stage": ["src", "core"] , "hdrs": ["util/avl.h"] , "deps": ["ref_counted", "useful", ["", "gpr_platform"], ["", "ref_counted_ptr"]] } , "time_averaged_stats": { "type": ["@", "rules", "CC", "library"] , "name": ["time_averaged_stats"] , "stage": ["src", "core"] , "srcs": ["util/time_averaged_stats.cc"] , "hdrs": ["util/time_averaged_stats.h"] , "deps": [["", "gpr"]] } , "forkable": { "type": ["@", "rules", "CC", "library"] , "name": ["forkable"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/forkable.cc"] , "hdrs": ["lib/event_engine/forkable.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["", "config_vars"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "event_engine_poller": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_poller"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/poller.h"] , "deps": [ ["@", "absl", "absl/functional", "function_ref"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "event_engine_time_util": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_time_util"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/time_util.cc"] , "hdrs": ["lib/event_engine/time_util.h"] , "deps": [["", "event_engine_base_hdrs"], ["", "gpr_platform"]] } , "event_engine_query_extensions": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_query_extensions"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/query_extensions.h"] , "deps": [["", "event_engine_base_hdrs"], ["", "gpr_platform"]] } , "event_engine_work_queue": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_work_queue"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/work_queue/work_queue.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "event_engine_basic_work_queue": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_basic_work_queue"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/work_queue/basic_work_queue.cc"] , "hdrs": ["lib/event_engine/work_queue/basic_work_queue.h"] , "deps": [ "common_event_engine_closures" , "event_engine_work_queue" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "common_event_engine_closures": { "type": ["@", "rules", "CC", "library"] , "name": ["common_event_engine_closures"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/common_closures.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "posix_event_engine_timer": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_timer"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/posix_engine/timer.cc" , "lib/event_engine/posix_engine/timer_heap.cc" ] , "hdrs": [ "lib/event_engine/posix_engine/timer.h" , "lib/event_engine/posix_engine/timer_heap.h" ] , "deps": [ "time" , "time_averaged_stats" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "event_engine_thread_local": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_thread_local"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/thread_local.cc"] , "hdrs": ["lib/event_engine/thread_local.h"] , "deps": [["", "gpr_platform"]] } , "event_engine_thread_count": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_thread_count"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/thread_pool/thread_count.cc"] , "hdrs": ["lib/event_engine/thread_pool/thread_count.h"] , "deps": [ "time" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/time", "time"] , ["", "gpr"] ] } , "event_engine_thread_pool": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_thread_pool"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/thread_pool/thread_pool_factory.cc" , "lib/event_engine/thread_pool/work_stealing_thread_pool.cc" ] , "hdrs": [ "lib/event_engine/thread_pool/thread_pool.h" , "lib/event_engine/thread_pool/work_stealing_thread_pool.h" ] , "deps": [ "common_event_engine_closures" , "env" , "event_engine_basic_work_queue" , "event_engine_thread_count" , "event_engine_thread_local" , "event_engine_work_queue" , "examine_stack" , "forkable" , "no_destruct" , "notification" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["", "backoff"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "posix_event_engine_base_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_base_hdrs"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix.h"] , "deps": [ "event_engine_extensions" , "event_engine_query_extensions" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "posix_event_engine_timer_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_timer_manager"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/timer_manager.cc"] , "hdrs": ["lib/event_engine/posix_engine/timer_manager.h"] , "deps": [ "event_engine_thread_pool" , "forkable" , "notification" , "posix_event_engine_timer" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "posix_event_engine_event_poller": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_event_poller"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/event_poller.h"] , "deps": [ "event_engine_poller" , "forkable" , "posix_event_engine_closure" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "posix_event_engine_closure": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_closure"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_closure.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "posix_event_engine_lockfree_event": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_lockfree_event"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/lockfree_event.cc"] , "hdrs": ["lib/event_engine/posix_engine/lockfree_event.h"] , "deps": [ "gpr_atm" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "status_helper" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["", "gpr"] ] } , "posix_event_engine_wakeup_fd_posix": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_wakeup_fd_posix"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_posix.h"] , "deps": [["@", "absl", "absl/status", "status"], ["", "gpr_platform"]] } , "posix_event_engine_wakeup_fd_posix_pipe": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_wakeup_fd_posix_pipe"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_pipe.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_pipe.h"] , "deps": [ "iomgr_port" , "posix_event_engine_wakeup_fd_posix" , "strerror" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "posix_event_engine_wakeup_fd_posix_eventfd": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_wakeup_fd_posix_eventfd"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_eventfd.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_eventfd.h"] , "deps": [ "iomgr_port" , "posix_event_engine_wakeup_fd_posix" , "strerror" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "posix_event_engine_wakeup_fd_posix_default": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_wakeup_fd_posix_default"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_posix_default.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_posix_default.h"] , "deps": [ "iomgr_port" , "posix_event_engine_wakeup_fd_posix" , "posix_event_engine_wakeup_fd_posix_eventfd" , "posix_event_engine_wakeup_fd_posix_pipe" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "gpr_platform"] ] } , "posix_event_engine_poller_posix_epoll1": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_poller_posix_epoll1"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/ev_epoll1_linux.cc"] , "hdrs": ["lib/event_engine/posix_engine/ev_epoll1_linux.h"] , "deps": [ "event_engine_poller" , "event_engine_time_util" , "iomgr_port" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "posix_event_engine_internal_errqueue" , "posix_event_engine_lockfree_event" , "posix_event_engine_wakeup_fd_posix" , "posix_event_engine_wakeup_fd_posix_default" , "status_helper" , "strerror" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_public_hdrs"] ] } , "posix_event_engine_poller_posix_poll": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_poller_posix_poll"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/ev_poll_posix.cc"] , "hdrs": ["lib/event_engine/posix_engine/ev_poll_posix.h"] , "deps": [ "common_event_engine_closures" , "event_engine_poller" , "event_engine_time_util" , "iomgr_port" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "posix_event_engine_wakeup_fd_posix" , "posix_event_engine_wakeup_fd_posix_default" , "status_helper" , "strerror" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_public_hdrs"] ] } , "posix_event_engine_poller_posix_default": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_poller_posix_default"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/event_poller_posix_default.cc"] , "hdrs": ["lib/event_engine/posix_engine/event_poller_posix_default.h"] , "deps": [ "forkable" , "iomgr_port" , "no_destruct" , "posix_event_engine_event_poller" , "posix_event_engine_poller_posix_epoll1" , "posix_event_engine_poller_posix_poll" , ["@", "absl", "absl/strings", "strings"] , ["", "config_vars"] , ["", "gpr"] ] } , "posix_event_engine_internal_errqueue": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_internal_errqueue"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/internal_errqueue.cc"] , "hdrs": ["lib/event_engine/posix_engine/internal_errqueue.h"] , "deps": ["iomgr_port", "strerror", ["@", "absl", "absl/log", "log"], ["", "gpr"]] } , "posix_event_engine_traced_buffer_list": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_traced_buffer_list"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/traced_buffer_list.cc"] , "hdrs": ["lib/event_engine/posix_engine/traced_buffer_list.h"] , "deps": [ "iomgr_port" , "posix_event_engine_internal_errqueue" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] ] } , "posix_event_engine_endpoint": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_endpoint"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_endpoint.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_endpoint.h"] , "deps": [ "event_engine_common" , "event_engine_extensions" , "event_engine_tcp_socket_utils" , "experiments" , "iomgr_port" , "load_file" , "memory_quota" , "posix_event_engine_base_hdrs" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "posix_event_engine_internal_errqueue" , "posix_event_engine_tcp_socket_utils" , "posix_event_engine_traced_buffer_list" , "ref_counted" , "resource_quota" , "slice" , "stats_data" , "status_helper" , "strerror" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] , ["", "stats"] ] } , "event_engine_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_utils"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/utils.cc"] , "hdrs": ["lib/event_engine/utils.h"] , "deps": [ "notification" , "time" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "posix_event_engine_tcp_socket_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_tcp_socket_utils"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/tcp_socket_utils.cc"] , "hdrs": ["lib/event_engine/posix_engine/tcp_socket_utils.h"] , "deps": [ "event_engine_tcp_socket_utils" , "iomgr_port" , "resource_quota" , "socket_mutator" , "status_helper" , "strerror" , "time" , "useful" , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "posix_event_engine_listener_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_listener_utils"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine_listener_utils.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_listener_utils.h"] , "deps": [ "event_engine_tcp_socket_utils" , "iomgr_port" , "posix_event_engine_tcp_socket_utils" , "socket_mutator" , "status_helper" , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "posix_event_engine_listener": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine_listener"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine_listener.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_listener.h"] , "deps": [ "event_engine_tcp_socket_utils" , "iomgr_port" , "posix_event_engine_base_hdrs" , "posix_event_engine_closure" , "posix_event_engine_endpoint" , "posix_event_engine_event_poller" , "posix_event_engine_listener_utils" , "posix_event_engine_tcp_socket_utils" , "socket_mutator" , "status_helper" , "strerror" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "posix_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["posix_event_engine"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine.h"] , "deps": [ "ares_resolver" , "event_engine_common" , "event_engine_poller" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_utils" , "experiments" , "forkable" , "init_internally" , "iomgr_port" , "native_posix_dns_resolver" , "no_destruct" , "posix_event_engine_base_hdrs" , "posix_event_engine_closure" , "posix_event_engine_endpoint" , "posix_event_engine_event_poller" , "posix_event_engine_listener" , "posix_event_engine_poller_posix_default" , "posix_event_engine_tcp_socket_utils" , "posix_event_engine_timer" , "posix_event_engine_timer_manager" , "ref_counted_dns_resolver_interface" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] , ["", "orphanable"] ] } , "windows_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["windows_event_engine"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/windows/windows_engine.cc"] , "hdrs": ["lib/event_engine/windows/windows_engine.h"] , "deps": [ "ares_resolver" , "channel_args_endpoint_config" , "common_event_engine_closures" , "dump_args" , "error" , "event_engine_common" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_utils" , "init_internally" , "iomgr_port" , "posix_event_engine_timer_manager" , "time" , "windows_endpoint" , "windows_event_engine_listener" , "windows_iocp" , "windows_native_resolver" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "windows_native_resolver": { "type": ["@", "rules", "CC", "library"] , "name": ["windows_native_resolver"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/windows/native_windows_dns_resolver.cc"] , "hdrs": ["lib/event_engine/windows/native_windows_dns_resolver.h"] , "deps": [ "error" , "status_helper" , ["@", "absl", "absl/strings", "str_format"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "windows_iocp": { "type": ["@", "rules", "CC", "library"] , "name": ["windows_iocp"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/windows/iocp.cc" , "lib/event_engine/windows/win_socket.cc" ] , "hdrs": [ "lib/event_engine/windows/iocp.h" , "lib/event_engine/windows/win_socket.h" ] , "deps": [ "error" , "event_engine_poller" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_time_util" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "windows_endpoint": { "type": ["@", "rules", "CC", "library"] , "name": ["windows_endpoint"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/windows/windows_endpoint.cc"] , "hdrs": ["lib/event_engine/windows/windows_endpoint.h"] , "deps": [ "error" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "status_helper" , "windows_iocp" , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "windows_event_engine_listener": { "type": ["@", "rules", "CC", "library"] , "name": ["windows_event_engine_listener"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/windows/windows_listener.cc"] , "hdrs": ["lib/event_engine/windows/windows_listener.h"] , "deps": [ "common_event_engine_closures" , "error" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "windows_endpoint" , "windows_iocp" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "cf_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["cf_event_engine"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/cf_engine/cf_engine.cc" , "lib/event_engine/cf_engine/cfstream_endpoint.cc" , "lib/event_engine/cf_engine/dns_service_resolver.cc" ] , "hdrs": [ "lib/event_engine/cf_engine/cf_engine.h" , "lib/event_engine/cf_engine/cfstream_endpoint.h" , "lib/event_engine/cf_engine/cftype_unique_ref.h" , "lib/event_engine/cf_engine/dns_service_resolver.h" ] , "deps": [ "event_engine_common" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_utils" , "init_internally" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "posix_event_engine_lockfree_event" , "posix_event_engine_timer_manager" , "ref_counted" , "strerror" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] ] } , "event_engine_tcp_socket_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_tcp_socket_utils"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/tcp_socket_utils.cc"] , "hdrs": ["lib/event_engine/tcp_socket_utils.h"] , "deps": [ "iomgr_port" , "resolved_address" , "status_helper" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "parse_address"] , ["", "uri"] ] } , "event_engine_shim": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_shim"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/shim.cc"] , "hdrs": ["lib/event_engine/shim.h"] , "deps": ["experiments", "iomgr_port", ["", "gpr_platform"]] } , "default_event_engine_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["default_event_engine_factory"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/default_event_engine_factory.cc"] , "hdrs": ["lib/event_engine/default_event_engine_factory.h"] , "deps": [ "posix_event_engine" , ["@", "absl", "absl/memory", "memory"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "channel_args_endpoint_config": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_args_endpoint_config"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/channel_args_endpoint_config.cc"] , "hdrs": ["lib/event_engine/channel_args_endpoint_config.h"] , "deps": [ "channel_args" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "thready_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["thready_event_engine"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/thready_event_engine/thready_event_engine.cc"] , "hdrs": ["lib/event_engine/thready_event_engine/thready_event_engine.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "event_engine_context": { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_context"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/event_engine_context.h"] , "deps": ["arena", ["", "event_engine_base_hdrs"], ["", "gpr"]] } , "default_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["default_event_engine"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/default_event_engine.cc"] , "hdrs": ["lib/event_engine/default_event_engine.h"] , "deps": [ "channel_args" , "default_event_engine_factory" , "no_destruct" , "thready_event_engine" , ["@", "absl", "absl/functional", "any_invocable"] , ["", "config"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "ref_counted_dns_resolver_interface": { "type": ["@", "rules", "CC", "library"] , "name": ["ref_counted_dns_resolver_interface"] , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/ref_counted_dns_resolver_interface.h"] , "deps": [ ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] , ["", "orphanable"] ] } , "native_posix_dns_resolver": { "type": ["@", "rules", "CC", "library"] , "name": ["native_posix_dns_resolver"] , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/native_posix_dns_resolver.cc"] , "hdrs": ["lib/event_engine/posix_engine/native_posix_dns_resolver.h"] , "deps": [ "iomgr_port" , "useful" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] ] } , "ares_resolver": { "type": ["@", "rules", "CC", "library"] , "name": ["ares_resolver"] , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/ares_resolver.cc" , "lib/event_engine/windows/grpc_polled_fd_windows.cc" ] , "hdrs": [ "lib/event_engine/ares_resolver.h" , "lib/event_engine/grpc_polled_fd.h" , "lib/event_engine/nameser.h" , "lib/event_engine/posix_engine/grpc_polled_fd_posix.h" , "lib/event_engine/windows/grpc_polled_fd_windows.h" ] , "deps": [ "common_event_engine_closures" , "error" , "event_engine_time_util" , "grpc_sockaddr" , "iomgr_port" , "posix_event_engine_closure" , "posix_event_engine_event_poller" , "posix_event_engine_tcp_socket_utils" , "ref_counted_dns_resolver_interface" , "resolved_address" , "slice" , "windows_iocp" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["@", "cares", "", "ares"] , ["", "config_vars"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["third_party/address_sorting", "address_sorting"] ] } , "channel_args_preconditioning": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_args_preconditioning"] , "stage": ["src", "core"] , "srcs": ["lib/channel/channel_args_preconditioning.cc"] , "hdrs": ["lib/channel/channel_args_preconditioning.h"] , "deps": ["channel_args", ["", "event_engine_base_hdrs"], ["", "gpr_platform"]] } , "bdp_estimator": { "type": ["@", "rules", "CC", "library"] , "name": ["bdp_estimator"] , "stage": ["src", "core"] , "srcs": ["lib/transport/bdp_estimator.cc"] , "hdrs": ["lib/transport/bdp_estimator.h"] , "deps": [ "time" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "percent_encoding": { "type": ["@", "rules", "CC", "library"] , "name": ["percent_encoding"] , "stage": ["src", "core"] , "srcs": ["lib/slice/percent_encoding.cc"] , "hdrs": ["lib/slice/percent_encoding.h"] , "deps": ["bitset", "slice", ["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "socket_mutator": { "type": ["@", "rules", "CC", "library"] , "name": ["socket_mutator"] , "stage": ["src", "core"] , "srcs": ["lib/iomgr/socket_mutator.cc"] , "hdrs": ["lib/iomgr/socket_mutator.h"] , "deps": ["channel_args", "useful", ["", "event_engine_base_hdrs"], ["", "gpr"]] } , "pollset_set": { "type": ["@", "rules", "CC", "library"] , "name": ["pollset_set"] , "stage": ["src", "core"] , "srcs": ["lib/iomgr/pollset_set.cc"] , "hdrs": ["lib/iomgr/pollset_set.h"] , "deps": ["iomgr_fwd", ["", "gpr"]] } , "histogram_view": { "type": ["@", "rules", "CC", "library"] , "name": ["histogram_view"] , "stage": ["src", "core"] , "srcs": ["telemetry/histogram_view.cc"] , "hdrs": ["telemetry/histogram_view.h"] , "deps": [["", "gpr"]] } , "stats_data": { "type": ["@", "rules", "CC", "library"] , "name": ["stats_data"] , "stage": ["src", "core"] , "srcs": ["telemetry/stats_data.cc"] , "hdrs": ["telemetry/stats_data.h"] , "deps": [ "histogram_view" , "per_cpu" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "per_cpu": { "type": ["@", "rules", "CC", "library"] , "name": ["per_cpu"] , "stage": ["src", "core"] , "srcs": ["util/per_cpu.cc"] , "hdrs": ["util/per_cpu.h"] , "deps": ["useful", ["", "gpr"]] } , "event_log": { "type": ["@", "rules", "CC", "library"] , "name": ["event_log"] , "stage": ["src", "core"] , "srcs": ["util/event_log.cc"] , "hdrs": ["util/event_log.h"] , "deps": [ "per_cpu" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "span"] , ["", "gpr"] ] } , "load_file": { "type": ["@", "rules", "CC", "library"] , "name": ["load_file"] , "stage": ["src", "core"] , "srcs": ["util/load_file.cc"] , "hdrs": ["util/load_file.h"] , "deps": [ "slice" , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "http2_errors": { "type": ["@", "rules", "CC", "library"] , "name": ["http2_errors"] , "stage": ["src", "core"] , "hdrs": ["lib/transport/http2_errors.h"] } , "channel_stack_type": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_stack_type"] , "stage": ["src", "core"] , "srcs": ["lib/surface/channel_stack_type.cc"] , "hdrs": ["lib/surface/channel_stack_type.h"] , "deps": [["", "gpr_platform"]] } , "channel_init": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_init"] , "stage": ["src", "core"] , "srcs": ["lib/surface/channel_init.cc"] , "hdrs": ["lib/surface/channel_init.h"] , "deps": [ "call_filters" , "channel_args" , "channel_fwd" , "channel_stack_type" , "interception_chain" , "unique_type_name" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_stack_builder"] , ["", "debug_location"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "server_interface": { "type": ["@", "rules", "CC", "library"] , "name": ["server_interface"] , "stage": ["src", "core"] , "hdrs": ["server/server_interface.h"] , "deps": [ "channel_args" , ["", "channelz"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] ] } , "single_set_ptr": { "type": ["@", "rules", "CC", "library"] , "name": ["single_set_ptr"] , "stage": ["src", "core"] , "hdrs": ["util/single_set_ptr.h"] , "deps": [["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "grpc_service_config": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_service_config"] , "stage": ["src", "core"] , "hdrs": [ "service_config/service_config.h" , "service_config/service_config_call_data.h" ] , "deps": [ "arena" , "chunked_vector" , "down_cast" , "ref_counted" , "service_config_parser" , "slice_refcount" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "service_config_parser": { "type": ["@", "rules", "CC", "library"] , "name": ["service_config_parser"] , "stage": ["src", "core"] , "srcs": ["service_config/service_config_parser.cc"] , "hdrs": ["service_config/service_config_parser.h"] , "deps": [ "channel_args" , "json" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "notification": { "type": ["@", "rules", "CC", "library"] , "name": ["notification"] , "stage": ["src", "core"] , "hdrs": ["util/notification.h"] , "deps": [["@", "absl", "absl/time", "time"], ["", "gpr"]] } , "channel_args": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_args"] , "stage": ["src", "core"] , "srcs": ["lib/channel/channel_args.cc"] , "hdrs": ["lib/channel/channel_args.h"] , "deps": [ "avl" , "channel_stack_type" , "dual_ref_counted" , "ref_counted" , "ref_counted_string" , "time" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "resolved_address": { "type": ["@", "rules", "CC", "library"] , "name": ["resolved_address"] , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/resolved_address.h"] , "deps": ["iomgr_port", ["", "gpr_platform"]] } , "client_channel_internal_header": { "type": ["@", "rules", "CC", "library"] , "name": ["client_channel_internal_header"] , "stage": ["src", "core"] , "hdrs": ["client_channel/client_channel_internal.h"] , "deps": [ "arena" , "call_destination" , "down_cast" , "grpc_service_config" , "lb_policy" , "unique_type_name" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["", "call_tracer"] , ["", "gpr"] ] } , "blackboard": { "type": ["@", "rules", "CC", "library"] , "name": ["blackboard"] , "stage": ["src", "core"] , "srcs": ["filter/blackboard.cc"] , "hdrs": ["filter/blackboard.h"] , "deps": [ "ref_counted" , "unique_type_name" , "useful" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/strings", "strings"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "ref_counted_ptr"] ] } , "filter_args": { "type": ["@", "rules", "CC", "library"] , "name": ["filter_args"] , "stage": ["src", "core"] , "hdrs": ["filter/filter_args.h"] , "deps": ["blackboard", "channel_fwd", "match"] } , "subchannel_connector": { "type": ["@", "rules", "CC", "library"] , "name": ["subchannel_connector"] , "stage": ["src", "core"] , "hdrs": ["client_channel/connector.h"] , "deps": [ "channel_args" , "closure" , "error" , "iomgr_fwd" , "resolved_address" , "time" , ["", "channelz"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "subchannel_pool_interface": { "type": ["@", "rules", "CC", "library"] , "name": ["subchannel_pool_interface"] , "stage": ["src", "core"] , "srcs": ["client_channel/subchannel_pool_interface.cc"] , "hdrs": ["client_channel/subchannel_pool_interface.h"] , "deps": [ "channel_args" , "ref_counted" , "resolved_address" , "useful" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] ] } , "config_selector": { "type": ["@", "rules", "CC", "library"] , "name": ["config_selector"] , "stage": ["src", "core"] , "hdrs": ["client_channel/config_selector.h"] , "deps": [ "arena" , "channel_args" , "channel_fwd" , "client_channel_internal_header" , "grpc_service_config" , "interception_chain" , "metadata_batch" , "ref_counted" , "slice" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_public_hdrs"] , ["", "grpc_public_hdrs"] , ["", "ref_counted_ptr"] ] } , "client_channel_service_config": { "type": ["@", "rules", "CC", "library"] , "name": ["client_channel_service_config"] , "stage": ["src", "core"] , "srcs": ["client_channel/client_channel_service_config.cc"] , "hdrs": ["client_channel/client_channel_service_config.h"] , "deps": [ "channel_args" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_registry" , "service_config_parser" , "time" , "validation_errors" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "client_channel_args": { "type": ["@", "rules", "CC", "library"] , "name": ["client_channel_args"] , "stage": ["src", "core"] , "hdrs": ["client_channel/client_channel_args.h"] } , "retry_interceptor": { "type": ["@", "rules", "CC", "library"] , "name": ["retry_interceptor"] , "stage": ["src", "core"] , "srcs": ["client_channel/retry_interceptor.cc"] , "hdrs": ["client_channel/retry_interceptor.h"] , "deps": [ "client_channel_args" , "filter_args" , "for_each" , "grpc_service_config" , "interception_chain" , "map" , "request_buffer" , "retry_service_config" , "retry_throttle" , "sleep" , ["", "backoff"] ] } , "retry_service_config": { "type": ["@", "rules", "CC", "library"] , "name": ["retry_service_config"] , "stage": ["src", "core"] , "srcs": ["client_channel/retry_service_config.cc"] , "hdrs": ["client_channel/retry_service_config.h"] , "deps": [ "channel_args" , "json" , "json_args" , "json_channel_args" , "json_object_loader" , "service_config_parser" , "time" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr_public_hdrs"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] ] } , "retry_throttle": { "type": ["@", "rules", "CC", "library"] , "name": ["retry_throttle"] , "stage": ["src", "core"] , "srcs": ["client_channel/retry_throttle.cc"] , "hdrs": ["client_channel/retry_throttle.h"] , "deps": [ "ref_counted" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "client_channel_backup_poller": { "type": ["@", "rules", "CC", "library"] , "name": ["client_channel_backup_poller"] , "stage": ["src", "core"] , "srcs": ["client_channel/backup_poller.cc"] , "hdrs": ["client_channel/backup_poller.h"] , "deps": [ "closure" , "error" , "iomgr_fwd" , "pollset_set" , "time" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["", "config_vars"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "iomgr"] , ["", "iomgr_timer"] ] } , "service_config_channel_arg_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["service_config_channel_arg_filter"] , "stage": ["src", "core"] , "srcs": ["service_config/service_config_channel_arg_filter.cc"] , "deps": [ "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "grpc_message_size_filter" , "grpc_service_config" , "latent_see" , "metadata_batch" , "service_config_parser" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr_platform"] , ["", "gpr_public_hdrs"] , ["", "grpc_base"] , ["", "grpc_service_config_impl"] , ["", "ref_counted_ptr"] ] } , "lb_policy": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_policy"] , "stage": ["src", "core"] , "srcs": ["load_balancing/lb_policy.cc"] , "hdrs": ["load_balancing/lb_policy.h"] , "deps": [ "channel_args" , "closure" , "dual_ref_counted" , "error" , "grpc_backend_metric_data" , "iomgr_fwd" , "metrics" , "pollset_set" , "ref_counted" , "resolved_address" , "subchannel_interface" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "lb_policy_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_policy_factory"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/lb_policy_factory.h"] , "deps": [ "json" , "lb_policy" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "lb_policy_registry": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_policy_registry"] , "stage": ["src", "core"] , "srcs": ["load_balancing/lb_policy_registry.cc"] , "hdrs": ["load_balancing/lb_policy_registry.h"] , "deps": [ "json" , "lb_policy" , "lb_policy_factory" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "lb_metadata": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_metadata"] , "stage": ["src", "core"] , "srcs": ["client_channel/lb_metadata.cc"] , "hdrs": ["client_channel/lb_metadata.h"] , "deps": [ "event_engine_common" , "lb_policy" , "metadata_batch" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "connection_context": { "type": ["@", "rules", "CC", "library"] , "name": ["connection_context"] , "stage": ["src", "core"] , "srcs": ["lib/surface/connection_context.cc"] , "hdrs": ["lib/surface/connection_context.h"] , "deps": ["no_destruct", ["", "gpr"], ["", "gpr_platform"], ["", "orphanable"]] } , "subchannel_interface": { "type": ["@", "rules", "CC", "library"] , "name": ["subchannel_interface"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/subchannel_interface.h"] , "deps": [ "dual_ref_counted" , "iomgr_fwd" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "delegating_helper": { "type": ["@", "rules", "CC", "library"] , "name": ["delegating_helper"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/delegating_helper.h"] , "deps": [ "channel_args" , "lb_policy" , "resolved_address" , "subchannel_interface" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "gpr_platform"] , ["", "grpc_security_base"] , ["", "ref_counted_ptr"] ] } , "backend_metric_parser": { "type": ["@", "rules", "CC", "library"] , "name": ["backend_metric_parser"] , "stage": ["src", "core"] , "srcs": ["load_balancing/backend_metric_parser.cc"] , "hdrs": ["load_balancing/backend_metric_parser.h"] , "deps": [ "grpc_backend_metric_data" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] , ["third_party/upb", "message"] ] } , "proxy_mapper": { "type": ["@", "rules", "CC", "library"] , "name": ["proxy_mapper"] , "stage": ["src", "core"] , "hdrs": ["handshaker/proxy_mapper.h"] , "deps": [ "channel_args" , "resolved_address" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr_platform"] ] } , "proxy_mapper_registry": { "type": ["@", "rules", "CC", "library"] , "name": ["proxy_mapper_registry"] , "stage": ["src", "core"] , "srcs": ["handshaker/proxy_mapper_registry.cc"] , "hdrs": ["handshaker/proxy_mapper_registry.h"] , "deps": [ "channel_args" , "proxy_mapper" , "resolved_address" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr_platform"] ] } , "http_proxy_mapper": { "type": ["@", "rules", "CC", "library"] , "name": ["http_proxy_mapper"] , "stage": ["src", "core"] , "srcs": ["handshaker/http_connect/http_proxy_mapper.cc"] , "hdrs": ["handshaker/http_connect/http_proxy_mapper.h"] , "deps": [ "channel_args" , "env" , "experiments" , "proxy_mapper" , "resolved_address" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "http_connect_handshaker"] , ["", "iomgr"] , ["", "parse_address"] , ["", "sockaddr_utils"] , ["", "uri"] ] } , "xds_http_proxy_mapper": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_http_proxy_mapper"] , "stage": ["src", "core"] , "srcs": ["handshaker/http_connect/xds_http_proxy_mapper.cc"] , "hdrs": ["handshaker/http_connect/xds_http_proxy_mapper.h"] , "deps": [ "channel_args" , "proxy_mapper" , "resolved_address" , "xds_endpoint" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "grpc_base"] , ["", "http_connect_handshaker"] , ["", "parse_address"] , ["", "sockaddr_utils"] ] } , "grpc_server_config_selector": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_server_config_selector"] , "stage": ["src", "core"] , "hdrs": ["server/server_config_selector.h"] , "deps": [ "dual_ref_counted" , "grpc_service_config" , "metadata_batch" , "ref_counted" , "service_config_parser" , "useful" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] ] } , "grpc_server_config_selector_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_server_config_selector_filter"] , "stage": ["src", "core"] , "srcs": ["server/server_config_selector_filter.cc"] , "hdrs": ["server/server_config_selector_filter.h"] , "deps": [ "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "context" , "event_engine_context" , "grpc_server_config_selector" , "grpc_service_config" , "latent_see" , "metadata_batch" , "status_helper" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_base"] , ["", "promise"] , ["", "ref_counted_ptr"] ] } , "sorted_pack": { "type": ["@", "rules", "CC", "library"] , "name": ["sorted_pack"] , "stage": ["src", "core"] , "hdrs": ["util/sorted_pack.h"] , "deps": ["type_list", ["", "gpr_platform"]] } , "type_list": { "type": ["@", "rules", "CC", "library"] , "name": ["type_list"] , "stage": ["src", "core"] , "hdrs": ["util/type_list.h"] } , "if_list": { "type": ["@", "rules", "CC", "library"] , "name": ["if_list"] , "stage": ["src", "core"] , "hdrs": ["util/if_list.h"] , "deps": [["", "gpr_platform"]] } , "tdigest": { "type": ["@", "rules", "CC", "library"] , "name": ["tdigest"] , "stage": ["src", "core"] , "srcs": ["util/tdigest.cc"] , "hdrs": ["util/tdigest.h"] , "deps": [ ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "certificate_provider_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["certificate_provider_factory"] , "stage": ["src", "core"] , "hdrs": ["lib/security/certificate_provider/certificate_provider_factory.h"] , "deps": [ "json" , "json_args" , "ref_counted" , "validation_errors" , ["@", "absl", "absl/strings", "strings"] , ["", "alts_util"] , ["", "gpr"] , ["", "grpc_core_credentials_header"] , ["", "ref_counted_ptr"] ] } , "certificate_provider_registry": { "type": ["@", "rules", "CC", "library"] , "name": ["certificate_provider_registry"] , "stage": ["src", "core"] , "srcs": ["lib/security/certificate_provider/certificate_provider_registry.cc"] , "hdrs": ["lib/security/certificate_provider/certificate_provider_registry.h"] , "deps": [ "certificate_provider_factory" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "grpc_audit_logging": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_audit_logging"] , "stage": ["src", "core"] , "srcs": [ "lib/security/authorization/audit_logging.cc" , "lib/security/authorization/stdout_logger.cc" ] , "hdrs": [ "lib/security/authorization/audit_logging.h" , "lib/security/authorization/stdout_logger.h" ] , "deps": [ ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["", "gpr"] , ["", "grpc_base"] ] } , "grpc_authorization_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_authorization_base"] , "stage": ["src", "core"] , "srcs": [ "lib/security/authorization/authorization_policy_provider_vtable.cc" , "lib/security/authorization/evaluate_args.cc" , "lib/security/authorization/grpc_server_authz_filter.cc" ] , "hdrs": [ "lib/security/authorization/authorization_engine.h" , "lib/security/authorization/authorization_policy_provider.h" , "lib/security/authorization/evaluate_args.h" , "lib/security/authorization/grpc_server_authz_filter.h" ] , "deps": [ "arena_promise" , "channel_args" , "channel_fwd" , "dual_ref_counted" , "endpoint_info_handshaker" , "latent_see" , "load_file" , "metadata_batch" , "ref_counted" , "resolved_address" , "slice" , "useful" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_credentials_util"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "parse_address"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "grpc_crl_provider": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_crl_provider"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/tls/grpc_tls_crl_provider.cc"] , "hdrs": ["lib/security/credentials/tls/grpc_tls_crl_provider.h"] , "deps": [ "default_event_engine" , "directory_reader" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] ] } , "grpc_fake_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_fake_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/fake/fake_credentials.cc" , "lib/security/security_connector/fake/fake_security_connector.cc" ] , "hdrs": [ "lib/security/credentials/fake/fake_credentials.h" , "lib/security/security_connector/fake/fake_security_connector.h" , "load_balancing/grpclb/grpclb.h" ] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "iomgr_fwd" , "metadata_batch" , "slice" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "resource_quota_api"] , ["", "tsi_base"] , ["", "tsi_fake_credentials"] ] } , "grpc_insecure_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_insecure_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/insecure/insecure_credentials.cc" , "lib/security/security_connector/insecure/insecure_security_connector.cc" ] , "hdrs": [ "lib/security/credentials/insecure/insecure_credentials.h" , "lib/security/security_connector/insecure/insecure_security_connector.h" ] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "iomgr_fwd" , "tsi_local_credentials" , "unique_type_name" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "tsi_base"] ] } , "tsi_local_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_local_credentials"] , "stage": ["src", "core"] , "srcs": ["tsi/local_transport_security.cc"] , "hdrs": ["tsi/local_transport_security.h"] , "deps": [ ["@", "absl", "absl/log", "log"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "tsi_base"] ] } , "grpc_local_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_local_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/local/local_credentials.cc" , "lib/security/security_connector/local/local_security_connector.cc" ] , "hdrs": [ "lib/security/credentials/local/local_credentials.h" , "lib/security/security_connector/local/local_security_connector.h" ] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "experiments" , "grpc_sockaddr" , "iomgr_fwd" , "resolved_address" , "tsi_local_credentials" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "parse_address"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "tsi_base"] , ["", "uri"] ] } , "grpc_ssl_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_ssl_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/ssl/ssl_credentials.cc" , "lib/security/security_connector/ssl/ssl_security_connector.cc" ] , "hdrs": [ "lib/security/credentials/ssl/ssl_credentials.h" , "lib/security/security_connector/ssl/ssl_security_connector.h" ] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "iomgr_fwd" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "handshaker"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "tsi_base"] , ["", "tsi_ssl_credentials"] , ["", "tsi_ssl_session_cache"] ] } , "grpc_google_default_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_google_default_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/google_default/credentials_generic.cc" , "lib/security/credentials/google_default/google_default_credentials.cc" ] , "hdrs": [ "lib/security/credentials/google_default/google_default_credentials.h" , "load_balancing/grpclb/grpclb.h" ] , "deps": [ "channel_args" , "closure" , "env" , "error" , "error_utils" , "grpc_external_account_credentials" , "grpc_lb_xds_channel_args" , "grpc_oauth2_credentials" , "grpc_ssl_credentials" , "iomgr_fwd" , "json" , "json_reader" , "load_file" , "slice" , "slice_refcount" , "status_helper" , "time" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "alts_util"] , ["", "channel_arg_names"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_alts_credentials"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_jwt_credentials"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "strerror": { "type": ["@", "rules", "CC", "library"] , "name": ["strerror"] , "stage": ["src", "core"] , "srcs": ["util/strerror.cc"] , "hdrs": ["util/strerror.h"] , "deps": [["@", "absl", "absl/strings", "str_format"], ["", "gpr_platform"]] } , "grpc_tls_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_tls_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/tls/grpc_tls_certificate_distributor.cc" , "lib/security/credentials/tls/grpc_tls_certificate_match.cc" , "lib/security/credentials/tls/grpc_tls_certificate_provider.cc" , "lib/security/credentials/tls/grpc_tls_certificate_verifier.cc" , "lib/security/credentials/tls/grpc_tls_credentials_options.cc" , "lib/security/credentials/tls/tls_credentials.cc" , "lib/security/security_connector/tls/tls_security_connector.cc" ] , "hdrs": [ "lib/security/credentials/tls/grpc_tls_certificate_distributor.h" , "lib/security/credentials/tls/grpc_tls_certificate_provider.h" , "lib/security/credentials/tls/grpc_tls_certificate_verifier.h" , "lib/security/credentials/tls/grpc_tls_credentials_options.h" , "lib/security/credentials/tls/tls_credentials.h" , "lib/security/security_connector/tls/tls_security_connector.h" ] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "iomgr_fwd" , "load_file" , "ref_counted" , "slice" , "slice_refcount" , "status_helper" , "unique_type_name" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "ssl", "", "crypto"] , ["@", "ssl", "", "ssl"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_credentials_util"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "handshaker"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "tsi_base"] , ["", "tsi_ssl_credentials"] , ["", "tsi_ssl_session_cache"] ] } , "grpc_iam_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_iam_credentials"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/iam/iam_credentials.cc"] , "hdrs": ["lib/security/credentials/iam/iam_credentials.h"] , "deps": [ "arena_promise" , "metadata_batch" , "slice" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "promise"] , ["", "ref_counted_ptr"] ] } , "token_fetcher_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["token_fetcher_credentials"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/token_fetcher/token_fetcher_credentials.cc"] , "hdrs": ["lib/security/credentials/token_fetcher/token_fetcher_credentials.h"] , "deps": [ "arena_promise" , "context" , "default_event_engine" , "metadata" , "poll" , "pollset_set" , "ref_counted" , "time" , "useful" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "variant"] , ["", "backoff"] , ["", "gpr"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "promise"] , ["", "ref_counted_ptr"] ] } , "gcp_service_account_identity_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["gcp_service_account_identity_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.cc" ] , "hdrs": [ "lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.h" ] , "deps": [ "activity" , "arena_promise" , "closure" , "error" , "json" , "json_args" , "json_object_loader" , "json_reader" , "metadata" , "pollset_set" , "ref_counted" , "slice" , "status_conversion" , "status_helper" , "time" , "token_fetcher_credentials" , "unique_type_name" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "grpc_oauth2_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_oauth2_credentials"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/oauth2/oauth2_credentials.cc"] , "hdrs": ["lib/security/credentials/oauth2/oauth2_credentials.h"] , "deps": [ "activity" , "arena_promise" , "closure" , "context" , "error" , "error_utils" , "httpcli_ssl_credentials" , "json" , "json_reader" , "load_file" , "metadata_batch" , "poll" , "pollset_set" , "ref_counted" , "slice" , "slice_refcount" , "status_helper" , "time" , "token_fetcher_credentials" , "unique_type_name" , "useful" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_credentials_util"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "grpc_external_account_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_external_account_credentials"] , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/external/aws_external_account_credentials.cc" , "lib/security/credentials/external/aws_request_signer.cc" , "lib/security/credentials/external/external_account_credentials.cc" , "lib/security/credentials/external/file_external_account_credentials.cc" , "lib/security/credentials/external/url_external_account_credentials.cc" ] , "hdrs": [ "lib/security/credentials/external/aws_external_account_credentials.h" , "lib/security/credentials/external/aws_request_signer.h" , "lib/security/credentials/external/external_account_credentials.h" , "lib/security/credentials/external/file_external_account_credentials.h" , "lib/security/credentials/external/url_external_account_credentials.h" ] , "deps": [ "closure" , "env" , "error" , "error_utils" , "grpc_oauth2_credentials" , "httpcli_ssl_credentials" , "json" , "json_reader" , "json_writer" , "load_file" , "slice" , "slice_refcount" , "status_helper" , "time" , "token_fetcher_credentials" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "ssl", "", "crypto"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_credentials_util"] , ["", "grpc_security_base"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "httpcli_ssl_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["httpcli_ssl_credentials"] , "stage": ["src", "core"] , "srcs": ["util/http_client/httpcli_security_connector.cc"] , "hdrs": ["util/http_client/httpcli_ssl_credentials.h"] , "deps": [ "arena_promise" , "channel_args" , "closure" , "error" , "iomgr_fwd" , "unique_type_name" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "tsi_base"] , ["", "tsi_ssl_credentials"] ] } , "tsi_ssl_types": { "type": ["@", "rules", "CC", "library"] , "name": ["tsi_ssl_types"] , "stage": ["src", "core"] , "hdrs": ["tsi/ssl_types.h"] , "deps": [["@", "ssl", "", "ssl"], ["", "gpr_platform"]] } , "grpc_matchers": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_matchers"] , "stage": ["src", "core"] , "srcs": ["util/matchers.cc"] , "hdrs": ["util/matchers.h"] , "deps": [ ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "re2", "", "re2"] , ["", "gpr"] ] } , "grpc_rbac_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_rbac_engine"] , "stage": ["src", "core"] , "srcs": [ "lib/security/authorization/grpc_authorization_engine.cc" , "lib/security/authorization/matchers.cc" , "lib/security/authorization/rbac_policy.cc" ] , "hdrs": [ "lib/security/authorization/grpc_authorization_engine.h" , "lib/security/authorization/matchers.h" , "lib/security/authorization/rbac_policy.h" ] , "deps": [ "grpc_audit_logging" , "grpc_authorization_base" , "grpc_matchers" , "resolved_address" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_base"] , ["", "parse_address"] , ["", "sockaddr_utils"] ] } , "json": { "type": ["@", "rules", "CC", "library"] , "name": ["json"] , "stage": ["src", "core"] , "hdrs": ["util/json/json.h"] , "deps": [["", "gpr"]] } , "json_reader": { "type": ["@", "rules", "CC", "library"] , "name": ["json_reader"] , "stage": ["src", "core"] , "srcs": ["util/json/json_reader.cc"] , "hdrs": ["util/json/json_reader.h"] , "deps": [ "json" , "match" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr"] ] } , "json_writer": { "type": ["@", "rules", "CC", "library"] , "name": ["json_writer"] , "stage": ["src", "core"] , "srcs": ["util/json/json_writer.cc"] , "hdrs": ["util/json/json_writer.h"] , "deps": ["json", ["@", "absl", "absl/strings", "strings"], ["", "gpr"]] } , "json_util": { "type": ["@", "rules", "CC", "library"] , "name": ["json_util"] , "stage": ["src", "core"] , "srcs": ["util/json/json_util.cc"] , "hdrs": ["util/json/json_util.h"] , "deps": [ "error" , "json" , "json_args" , "json_object_loader" , "no_destruct" , "time" , "validation_errors" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] ] } , "json_args": { "type": ["@", "rules", "CC", "library"] , "name": ["json_args"] , "stage": ["src", "core"] , "hdrs": ["util/json/json_args.h"] , "deps": [["@", "absl", "absl/strings", "strings"], ["", "gpr"]] } , "json_object_loader": { "type": ["@", "rules", "CC", "library"] , "name": ["json_object_loader"] , "stage": ["src", "core"] , "srcs": ["util/json/json_object_loader.cc"] , "hdrs": ["util/json/json_object_loader.h"] , "deps": [ "json" , "json_args" , "no_destruct" , "time" , "validation_errors" , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "ref_counted_ptr"] ] } , "json_channel_args": { "type": ["@", "rules", "CC", "library"] , "name": ["json_channel_args"] , "stage": ["src", "core"] , "hdrs": ["util/json/json_channel_args.h"] , "deps": [ "channel_args" , "json_args" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] ] } , "idle_filter_state": { "type": ["@", "rules", "CC", "library"] , "name": ["idle_filter_state"] , "stage": ["src", "core"] , "srcs": ["ext/filters/channel_idle/idle_filter_state.cc"] , "hdrs": ["ext/filters/channel_idle/idle_filter_state.h"] , "deps": [["", "gpr_platform"]] } , "grpc_channel_idle_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_channel_idle_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/channel_idle/legacy_channel_idle_filter.cc"] , "hdrs": ["ext/filters/channel_idle/legacy_channel_idle_filter.h"] , "deps": [ "activity" , "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "closure" , "connectivity_state" , "error" , "exec_ctx_wakeup_scheduler" , "experiments" , "http2_errors" , "idle_filter_state" , "loop" , "metadata_batch" , "no_destruct" , "per_cpu" , "poll" , "single_set_ptr" , "sleep" , "status_helper" , "time" , "try_seq" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "promise"] , ["", "ref_counted_ptr"] ] } , "grpc_client_authority_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_client_authority_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/http/client_authority_filter.cc"] , "hdrs": ["ext/filters/http/client_authority_filter.h"] , "deps": [ "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "latent_see" , "metadata_batch" , "slice" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_security_base"] ] } , "grpc_message_size_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_message_size_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/message_size/message_size_filter.cc"] , "hdrs": ["ext/filters/message_size/message_size_filter.h"] , "deps": [ "activity" , "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "latch" , "latent_see" , "metadata_batch" , "poll" , "race" , "service_config_parser" , "slice" , "slice_buffer" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] ] } , "grpc_fault_injection_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_fault_injection_filter"] , "stage": ["src", "core"] , "srcs": [ "ext/filters/fault_injection/fault_injection_filter.cc" , "ext/filters/fault_injection/fault_injection_service_config_parser.cc" ] , "hdrs": [ "ext/filters/fault_injection/fault_injection_filter.h" , "ext/filters/fault_injection/fault_injection_service_config_parser.h" ] , "deps": [ "arena_promise" , "channel_args" , "channel_fwd" , "context" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "metadata_batch" , "service_config_parser" , "sleep" , "time" , "try_seq" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] ] } , "grpc_rbac_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_rbac_filter"] , "stage": ["src", "core"] , "srcs": [ "ext/filters/rbac/rbac_filter.cc" , "ext/filters/rbac/rbac_service_config_parser.cc" ] , "hdrs": [ "ext/filters/rbac/rbac_filter.h" , "ext/filters/rbac/rbac_service_config_parser.h" ] , "deps": [ "arena_promise" , "channel_args" , "channel_fwd" , "context" , "error" , "grpc_audit_logging" , "grpc_authorization_base" , "grpc_matchers" , "grpc_rbac_engine" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "latent_see" , "metadata_batch" , "service_config_parser" , "validation_errors" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_security_base"] , ["", "promise"] ] } , "grpc_stateful_session_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_stateful_session_filter"] , "stage": ["src", "core"] , "srcs": [ "ext/filters/stateful_session/stateful_session_filter.cc" , "ext/filters/stateful_session/stateful_session_service_config_parser.cc" ] , "hdrs": [ "ext/filters/stateful_session/stateful_session_filter.h" , "ext/filters/stateful_session/stateful_session_service_config_parser.h" ] , "deps": [ "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "context" , "grpc_resolver_xds_attributes" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "latent_see" , "map" , "metadata_batch" , "pipe" , "ref_counted_string" , "service_config_parser" , "slice" , "time" , "unique_type_name" , "validation_errors" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] ] } , "gcp_authentication_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["gcp_authentication_filter"] , "stage": ["src", "core"] , "srcs": [ "ext/filters/gcp_authentication/gcp_authentication_filter.cc" , "ext/filters/gcp_authentication/gcp_authentication_service_config_parser.cc" ] , "hdrs": [ "ext/filters/gcp_authentication/gcp_authentication_filter.h" , "ext/filters/gcp_authentication/gcp_authentication_service_config_parser.h" ] , "deps": [ "arena" , "blackboard" , "channel_args" , "channel_fwd" , "context" , "gcp_service_account_identity_credentials" , "grpc_resolver_xds_attributes" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "lru_cache" , "service_config_parser" , "validation_errors" , "xds_config" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] ] } , "grpc_lb_policy_grpclb": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_grpclb"] , "stage": ["src", "core"] , "srcs": [ "load_balancing/grpclb/client_load_reporting_filter.cc" , "load_balancing/grpclb/grpclb.cc" , "load_balancing/grpclb/grpclb_client_stats.cc" , "load_balancing/grpclb/load_balancer_api.cc" ] , "hdrs": [ "load_balancing/grpclb/client_load_reporting_filter.h" , "load_balancing/grpclb/grpclb.h" , "load_balancing/grpclb/grpclb_client_stats.h" , "load_balancing/grpclb/load_balancer_api.h" ] , "deps": [ "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "closure" , "connectivity_state" , "context" , "delegating_helper" , "error" , "experiments" , "gpr_atm" , "grpc_sockaddr" , "json" , "json_args" , "json_object_loader" , "latent_see" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "map" , "metadata_batch" , "pipe" , "pollset_set" , "ref_counted" , "resolved_address" , "slice" , "slice_refcount" , "status_helper" , "subchannel_interface" , "time" , "useful" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "globals"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "backoff"] , ["", "channel"] , ["", "channel_arg_names"] , ["", "channel_create"] , ["", "channelz"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_grpclb_balancer_addresses"] , ["", "grpc_public_hdrs"] , ["", "grpc_resolver"] , ["", "grpc_resolver_fake"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "work_serializer"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "random_early_detection": { "type": ["@", "rules", "CC", "library"] , "name": ["random_early_detection"] , "stage": ["src", "core"] , "srcs": ["util/random_early_detection.cc"] , "hdrs": ["util/random_early_detection.h"] , "deps": [ ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "distributions"] , ["", "gpr_platform"] ] } , "grpc_backend_metric_data": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_backend_metric_data"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/backend_metric_data.h"] , "deps": [["@", "absl", "absl/strings", "strings"], ["", "gpr_platform"]] } , "grpc_backend_metric_provider": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_backend_metric_provider"] , "stage": ["src", "core"] , "hdrs": ["ext/filters/backend_metrics/backend_metric_provider.h"] , "deps": ["arena"] } , "grpc_lb_policy_rls": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_rls"] , "stage": ["src", "core"] , "srcs": ["load_balancing/rls/rls.cc"] , "hdrs": ["load_balancing/rls/rls.h"] , "deps": [ "channel_args" , "closure" , "connectivity_state" , "delegating_helper" , "dual_ref_counted" , "error" , "error_utils" , "grpc_fake_credentials" , "json" , "json_args" , "json_object_loader" , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "match" , "metrics" , "pollset_set" , "slice" , "slice_refcount" , "status_helper" , "time" , "upb_utils" , "uuid_v4" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "backoff"] , ["", "channel"] , ["", "channel_arg_names"] , ["", "channel_create"] , ["", "channelz"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_public_hdrs"] , ["", "grpc_resolver"] , ["", "grpc_security_base"] , ["", "grpc_service_config_impl"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "lru_cache": { "type": ["@", "rules", "CC", "library"] , "name": ["lru_cache"] , "stage": ["src", "core"] , "hdrs": ["util/lru_cache.h"] , "deps": [ ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/types", "optional"] , ["", "grpc_public_hdrs"] ] } , "upb_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["upb_utils"] , "stage": ["src", "core"] , "hdrs": ["util/upb_utils.h"] , "deps": [["@", "absl", "absl/strings", "strings"], ["third_party/upb", "base"]] } , "xds_enabled_server": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_enabled_server"] , "stage": ["src", "core"] , "hdrs": ["xds/grpc/xds_enabled_server.h"] } , "xds_certificate_provider": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_certificate_provider"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_certificate_provider.cc"] , "hdrs": ["xds/grpc/xds_certificate_provider.h"] , "deps": [ "channel_args" , "error" , "grpc_matchers" , "grpc_tls_credentials" , "unique_type_name" , "useful" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] , ["", "tsi_ssl_credentials"] ] } , "xds_certificate_provider_store": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_certificate_provider_store"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/certificate_provider_store.cc"] , "hdrs": ["xds/grpc/certificate_provider_store.h"] , "deps": [ "certificate_provider_factory" , "certificate_provider_registry" , "grpc_tls_credentials" , "json" , "json_args" , "json_object_loader" , "unique_type_name" , "useful" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "orphanable"] , ["", "ref_counted_ptr"] ] } , "xds_credentials": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_credentials"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/xds/xds_credentials.cc"] , "hdrs": ["lib/security/credentials/xds/xds_credentials.h"] , "deps": [ "channel_args" , "grpc_lb_xds_channel_args" , "grpc_matchers" , "grpc_tls_credentials" , "unique_type_name" , "useful" , "xds_certificate_provider" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_credentials_util"] , ["", "grpc_security_base"] , ["", "ref_counted_ptr"] ] } , "xds_file_watcher_certificate_provider_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_file_watcher_certificate_provider_factory"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/file_watcher_certificate_provider_factory.cc"] , "hdrs": ["xds/grpc/file_watcher_certificate_provider_factory.h"] , "deps": [ "certificate_provider_factory" , "grpc_tls_credentials" , "json" , "json_args" , "json_object_loader" , "time" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "gpr"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] ] } , "xds_common_types": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_common_types"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_common_types.cc"] , "hdrs": ["xds/grpc/xds_common_types.h"] , "deps": [ "grpc_matchers" , "json" , "match" , "validation_errors" , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] ] } , "xds_http_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_http_filter"] , "stage": ["src", "core"] , "hdrs": ["xds/grpc/xds_http_filter.h"] , "deps": [ "channel_args" , "channel_fwd" , "interception_chain" , "json" , "json_writer" , "validation_errors" , "xds_common_types" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "xds_client"] , ["third_party/upb", "reflection"] ] } , "xds_route_config": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_route_config"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_route_config.cc"] , "hdrs": ["xds/grpc/xds_route_config.h"] , "deps": [ "grpc_matchers" , "match" , "time" , "xds_http_filter" , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["@", "re2", "", "re2"] , ["", "grpc_base"] , ["", "xds_client"] ] } , "xds_listener": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_listener"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_listener.cc"] , "hdrs": ["xds/grpc/xds_listener.h"] , "deps": [ "match" , "resolved_address" , "time" , "xds_common_types" , "xds_http_filter" , "xds_route_config" , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "sockaddr_utils"] , ["", "xds_client"] ] } , "xds_health_status": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_health_status"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_health_status.cc"] , "hdrs": ["xds/grpc/xds_health_status.h"] , "deps": [ ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["", "endpoint_addresses"] , ["src/core/ext/upb-gen", "upb-gen-lib"] ] } , "xds_server_grpc": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_server_grpc"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_server_grpc.cc"] , "hdrs": ["xds/grpc/xds_server_grpc.h"] , "deps": [ "channel_creds_registry" , "json" , "json_args" , "json_object_loader" , "json_reader" , "json_writer" , "validation_errors" , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "ref_counted_ptr"] , ["", "xds_client"] ] } , "xds_metadata": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_metadata"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_metadata.cc"] , "hdrs": ["xds/grpc/xds_metadata.h"] , "deps": [ "down_cast" , "json" , "json_writer" , "validation_errors" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] ] } , "xds_cluster": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_cluster"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_cluster.cc"] , "hdrs": ["xds/grpc/xds_cluster.h"] , "deps": [ "grpc_outlier_detection_header" , "json" , "json_writer" , "match" , "time" , "xds_backend_metric_propagation" , "xds_common_types" , "xds_health_status" , "xds_metadata" , "xds_server_grpc" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "xds_client"] ] } , "xds_endpoint": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_endpoint"] , "stage": ["src", "core"] , "srcs": ["xds/grpc/xds_endpoint.cc"] , "hdrs": ["xds/grpc/xds_endpoint.h"] , "deps": [ "ref_counted" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/strings", "strings"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "ref_counted_ptr"] , ["", "xds_client"] ] } , "xds_backend_metric_propagation": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_backend_metric_propagation"] , "stage": ["src", "core"] , "srcs": ["xds/xds_client/xds_backend_metric_propagation.cc"] , "hdrs": ["xds/xds_client/xds_backend_metric_propagation.h"] , "deps": [ "ref_counted" , "useful" , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/strings", "strings"] , ["", "ref_counted_ptr"] ] } , "grpc_xds_client": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_xds_client"] , "stage": ["src", "core"] , "srcs": [ "xds/grpc/xds_audit_logger_registry.cc" , "xds/grpc/xds_bootstrap_grpc.cc" , "xds/grpc/xds_client_grpc.cc" , "xds/grpc/xds_cluster_parser.cc" , "xds/grpc/xds_cluster_specifier_plugin.cc" , "xds/grpc/xds_common_types_parser.cc" , "xds/grpc/xds_endpoint_parser.cc" , "xds/grpc/xds_http_fault_filter.cc" , "xds/grpc/xds_http_filter_registry.cc" , "xds/grpc/xds_http_gcp_authn_filter.cc" , "xds/grpc/xds_http_rbac_filter.cc" , "xds/grpc/xds_http_stateful_session_filter.cc" , "xds/grpc/xds_lb_policy_registry.cc" , "xds/grpc/xds_listener_parser.cc" , "xds/grpc/xds_metadata_parser.cc" , "xds/grpc/xds_route_config_parser.cc" , "xds/grpc/xds_routing.cc" , "xds/grpc/xds_transport_grpc.cc" ] , "hdrs": [ "xds/grpc/xds_audit_logger_registry.h" , "xds/grpc/xds_bootstrap_grpc.h" , "xds/grpc/xds_client_grpc.h" , "xds/grpc/xds_cluster_parser.h" , "xds/grpc/xds_cluster_specifier_plugin.h" , "xds/grpc/xds_common_types_parser.h" , "xds/grpc/xds_endpoint_parser.h" , "xds/grpc/xds_http_fault_filter.h" , "xds/grpc/xds_http_filter_registry.h" , "xds/grpc/xds_http_gcp_authn_filter.h" , "xds/grpc/xds_http_rbac_filter.h" , "xds/grpc/xds_http_stateful_session_filter.h" , "xds/grpc/xds_lb_policy_registry.h" , "xds/grpc/xds_listener_parser.h" , "xds/grpc/xds_metadata_parser.h" , "xds/grpc/xds_route_config_parser.h" , "xds/grpc/xds_routing.h" , "xds/grpc/xds_transport_grpc.h" ] , "deps": [ "certificate_provider_factory" , "certificate_provider_registry" , "channel_args" , "channel_args_endpoint_config" , "channel_creds_registry" , "channel_fwd" , "closure" , "connectivity_state" , "default_event_engine" , "down_cast" , "env" , "error" , "error_utils" , "gcp_authentication_filter" , "grpc_audit_logging" , "grpc_fake_credentials" , "grpc_fault_injection_filter" , "grpc_lb_policy_pick_first" , "grpc_lb_policy_ring_hash" , "grpc_lb_xds_channel_args" , "grpc_matchers" , "grpc_outlier_detection_header" , "grpc_rbac_filter" , "grpc_sockaddr" , "grpc_stateful_session_filter" , "grpc_tls_credentials" , "grpc_transport_chttp2_client_connector" , "init_internally" , "interception_chain" , "iomgr_fwd" , "json" , "json_args" , "json_object_loader" , "json_reader" , "json_util" , "json_writer" , "lb_policy_registry" , "load_file" , "match" , "metadata_batch" , "metrics" , "pollset_set" , "ref_counted" , "resolved_address" , "slice" , "slice_refcount" , "status_conversion" , "status_helper" , "time" , "unique_type_name" , "upb_utils" , "useful" , "validation_errors" , "xds_backend_metric_propagation" , "xds_certificate_provider" , "xds_certificate_provider_store" , "xds_cluster" , "xds_common_types" , "xds_credentials" , "xds_endpoint" , "xds_file_watcher_certificate_provider_factory" , "xds_health_status" , "xds_http_filter" , "xds_listener" , "xds_metadata" , "xds_route_config" , "xds_server_grpc" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] , ["@", "protobuf", "", "libprotobuf"] , ["@", "re2", "", "re2"] , ["", "channel"] , ["", "channel_arg_names"] , ["", "channel_create"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_core_credentials_header"] , ["", "grpc_credentials_util"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "iomgr"] , ["", "iomgr_timer"] , ["", "orphanable"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "tsi_ssl_credentials"] , ["", "uri"] , ["", "work_serializer"] , ["", "xds_client"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["src/core/ext/upbdefs-gen", "upbdefs-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "json"] , ["third_party/upb", "mem"] , ["third_party/upb", "message"] , ["third_party/upb", "reflection"] , ["third_party/upb", "text"] ] } , "grpc_xds_channel_stack_modifier": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_xds_channel_stack_modifier"] , "stage": ["src", "core"] , "srcs": ["server/xds_channel_stack_modifier.cc"] , "hdrs": ["server/xds_channel_stack_modifier.h"] , "deps": [ "channel_args" , "channel_fwd" , "channel_init" , "channel_stack_type" , "ref_counted" , "useful" , ["@", "absl", "absl/strings", "strings"] , ["", "channel_stack_builder"] , ["", "config"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] ] } , "grpc_xds_server_config_fetcher": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_xds_server_config_fetcher"] , "stage": ["src", "core"] , "srcs": ["server/xds_server_config_fetcher.cc"] , "deps": [ "channel_args" , "channel_args_preconditioning" , "channel_fwd" , "grpc_server_config_selector" , "grpc_server_config_selector_filter" , "grpc_service_config" , "grpc_sockaddr" , "grpc_tls_credentials" , "grpc_xds_channel_stack_modifier" , "grpc_xds_client" , "iomgr_fwd" , "match" , "metadata_batch" , "resolved_address" , "slice_refcount" , "unique_type_name" , "xds_certificate_provider" , "xds_certificate_provider_store" , "xds_common_types" , "xds_credentials" , "xds_http_filter" , "xds_listener" , "xds_route_config" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] , ["", "grpc_service_config_impl"] , ["", "grpc_trace"] , ["", "iomgr"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "server"] , ["", "sockaddr_utils"] , ["", "uri"] , ["", "xds_client"] ] } , "channel_creds_registry_init": { "type": ["@", "rules", "CC", "library"] , "name": ["channel_creds_registry_init"] , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/channel_creds_registry_init.cc"] , "deps": [ "channel_creds_registry" , "grpc_fake_credentials" , "grpc_google_default_credentials" , "grpc_tls_credentials" , "json" , "json_args" , "json_object_loader" , "time" , "validation_errors" , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "ref_counted_ptr"] ] } , "grpc_lb_policy_cds": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_cds"] , "stage": ["src", "core"] , "srcs": ["load_balancing/xds/cds.cc"] , "deps": [ "channel_args" , "delegating_helper" , "env" , "grpc_lb_address_filtering" , "grpc_lb_xds_channel_args" , "grpc_outlier_detection_header" , "grpc_xds_client" , "json" , "json_args" , "json_object_loader" , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "match" , "pollset_set" , "time" , "unique_type_name" , "xds_cluster" , "xds_common_types" , "xds_config" , "xds_dependency_manager" , "xds_health_status" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "config"] , ["", "debug_location"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] , ["", "xds_client"] ] } , "grpc_lb_xds_channel_args": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_xds_channel_args"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/xds/xds_channel_args.h"] , "deps": [["", "endpoint_addresses"], ["", "gpr_platform"]] } , "grpc_lb_policy_xds_cluster_impl": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_xds_cluster_impl"] , "stage": ["src", "core"] , "srcs": ["load_balancing/xds/xds_cluster_impl.cc"] , "deps": [ "channel_args" , "client_channel_internal_header" , "connectivity_state" , "delegating_helper" , "grpc_backend_metric_data" , "grpc_lb_xds_channel_args" , "grpc_resolver_xds_attributes" , "grpc_xds_client" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "match" , "pollset_set" , "ref_counted" , "ref_counted_string" , "resolved_address" , "subchannel_interface" , "validation_errors" , "xds_config" , "xds_credentials" , "xds_endpoint" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "call_tracer"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "xds_client"] ] } , "grpc_lb_policy_xds_cluster_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_xds_cluster_manager"] , "stage": ["src", "core"] , "srcs": ["load_balancing/xds/xds_cluster_manager.cc"] , "deps": [ "channel_args" , "client_channel_internal_header" , "connectivity_state" , "delegating_helper" , "grpc_resolver_xds_attributes" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "time" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "grpc_lb_policy_xds_wrr_locality": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_xds_wrr_locality"] , "stage": ["src", "core"] , "srcs": ["load_balancing/xds/xds_wrr_locality.cc"] , "deps": [ "channel_args" , "delegating_helper" , "grpc_lb_xds_channel_args" , "json" , "json_args" , "json_object_loader" , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "ref_counted_string" , "validation_errors" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "xds_client"] ] } , "grpc_lb_address_filtering": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_address_filtering"] , "stage": ["src", "core"] , "srcs": ["load_balancing/address_filtering.cc"] , "hdrs": ["load_balancing/address_filtering.h"] , "deps": [ "channel_args" , "ref_counted" , "ref_counted_string" , "resolved_address" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "endpoint_addresses"] , ["", "gpr_platform"] , ["", "ref_counted_ptr"] ] } , "health_check_client": { "type": ["@", "rules", "CC", "library"] , "name": ["health_check_client"] , "stage": ["src", "core"] , "srcs": ["load_balancing/health_check_client.cc"] , "hdrs": [ "load_balancing/health_check_client.h" , "load_balancing/health_check_client_internal.h" ] , "deps": [ "channel_args" , "client_channel_internal_header" , "closure" , "connectivity_state" , "error" , "iomgr_fwd" , "pollset_set" , "slice" , "subchannel_interface" , "unique_type_name" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "channelz"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "work_serializer"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "lb_endpoint_list": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_endpoint_list"] , "stage": ["src", "core"] , "srcs": ["load_balancing/endpoint_list.cc"] , "hdrs": ["load_balancing/endpoint_list.h"] , "deps": [ "channel_args" , "delegating_helper" , "down_cast" , "grpc_lb_policy_pick_first" , "json" , "lb_policy" , "lb_policy_registry" , "pollset_set" , "resolved_address" , "subchannel_interface" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "grpc_lb_policy_pick_first": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_pick_first"] , "stage": ["src", "core"] , "srcs": ["load_balancing/pick_first/pick_first.cc"] , "hdrs": ["load_balancing/pick_first/pick_first.h"] , "deps": [ "channel_args" , "connectivity_state" , "experiments" , "grpc_outlier_detection_header" , "health_check_client" , "iomgr_fwd" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "metrics" , "resolved_address" , "subchannel_interface" , "time" , "useful" , ["@", "absl", "absl/algorithm", "container"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "work_serializer"] ] } , "down_cast": { "type": ["@", "rules", "CC", "library"] , "name": ["down_cast"] , "stage": ["src", "core"] , "hdrs": ["util/down_cast.h"] , "deps": [ ["@", "absl", "absl/base", "config"] , ["@", "absl", "absl/log", "check"] , ["", "gpr"] ] } , "glob": { "type": ["@", "rules", "CC", "library"] , "name": ["glob"] , "stage": ["src", "core"] , "srcs": ["util/glob.cc"] , "hdrs": ["util/glob.h"] , "deps": [["@", "absl", "absl/strings", "strings"]] } , "status_conversion": { "type": ["@", "rules", "CC", "library"] , "name": ["status_conversion"] , "stage": ["src", "core"] , "srcs": ["lib/transport/status_conversion.cc"] , "hdrs": ["lib/transport/status_conversion.h"] , "deps": ["http2_errors", "time", ["", "gpr_platform"], ["", "grpc_public_hdrs"]] } , "error_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["error_utils"] , "stage": ["src", "core"] , "srcs": ["lib/transport/error_utils.cc"] , "hdrs": ["lib/transport/error_utils.h"] , "deps": [ "error" , "http2_errors" , "status_conversion" , "status_helper" , "time" , ["@", "absl", "absl/status", "status"] , ["", "gpr_platform"] , ["", "grpc_public_hdrs"] ] } , "connectivity_state": { "type": ["@", "rules", "CC", "library"] , "name": ["connectivity_state"] , "stage": ["src", "core"] , "srcs": ["lib/transport/connectivity_state.cc"] , "hdrs": ["lib/transport/connectivity_state.h"] , "deps": [ "closure" , "error" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "xxhash_inline": { "type": ["@", "rules", "CC", "library"] , "name": ["xxhash_inline"] , "stage": ["src", "core"] , "hdrs": ["util/xxhash_inline.h"] , "deps": [["", "gpr_platform"], ["third_party/xxhash", "xxhash"]] } , "grpc_lb_policy_ring_hash": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_ring_hash"] , "stage": ["src", "core"] , "srcs": ["load_balancing/ring_hash/ring_hash.cc"] , "hdrs": ["load_balancing/ring_hash/ring_hash.h"] , "deps": [ "channel_args" , "client_channel_internal_header" , "closure" , "connectivity_state" , "delegating_helper" , "env" , "error" , "grpc_lb_policy_pick_first" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "ref_counted" , "ref_counted_string" , "resolved_address" , "subchannel_interface" , "unique_type_name" , "validation_errors" , "xxhash_inline" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "work_serializer"] , ["third_party/xxhash", "xxhash"] ] } , "grpc_lb_policy_round_robin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_round_robin"] , "stage": ["src", "core"] , "srcs": ["load_balancing/round_robin/round_robin.cc"] , "deps": [ "channel_args" , "connectivity_state" , "json" , "lb_endpoint_list" , "lb_policy" , "lb_policy_factory" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "static_stride_scheduler": { "type": ["@", "rules", "CC", "library"] , "name": ["static_stride_scheduler"] , "stage": ["src", "core"] , "srcs": ["load_balancing/weighted_round_robin/static_stride_scheduler.cc"] , "hdrs": ["load_balancing/weighted_round_robin/static_stride_scheduler.h"] , "deps": [ ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["", "gpr"] ] } , "grpc_lb_policy_weighted_round_robin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_weighted_round_robin"] , "stage": ["src", "core"] , "srcs": ["load_balancing/weighted_round_robin/weighted_round_robin.cc"] , "deps": [ "channel_args" , "connectivity_state" , "experiments" , "grpc_backend_metric_data" , "grpc_lb_policy_weighted_target" , "json" , "json_args" , "json_object_loader" , "lb_endpoint_list" , "lb_policy" , "lb_policy_factory" , "metrics" , "ref_counted" , "resolved_address" , "static_stride_scheduler" , "stats_data" , "subchannel_interface" , "time" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "oob_backend_metric"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "stats"] , ["", "work_serializer"] ] } , "grpc_outlier_detection_header": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_outlier_detection_header"] , "stage": ["src", "core"] , "hdrs": ["load_balancing/outlier_detection/outlier_detection.h"] , "deps": [ "json" , "json_args" , "json_object_loader" , "time" , "validation_errors" , ["@", "absl", "absl/types", "optional"] , ["", "gpr_platform"] , ["", "server_address"] ] } , "grpc_lb_policy_outlier_detection": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_outlier_detection"] , "stage": ["src", "core"] , "srcs": ["load_balancing/outlier_detection/outlier_detection.cc"] , "deps": [ "channel_args" , "connectivity_state" , "delegating_helper" , "experiments" , "grpc_outlier_detection_header" , "health_check_client" , "iomgr_fwd" , "json" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "ref_counted" , "resolved_address" , "subchannel_interface" , "unique_type_name" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] , ["", "work_serializer"] ] } , "grpc_lb_policy_priority": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_priority"] , "stage": ["src", "core"] , "srcs": ["load_balancing/priority/priority.cc"] , "deps": [ "channel_args" , "connectivity_state" , "delegating_helper" , "grpc_lb_address_filtering" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "ref_counted_string" , "time" , "validation_errors" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "grpc_lb_policy_weighted_target": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_weighted_target"] , "stage": ["src", "core"] , "srcs": ["load_balancing/weighted_target/weighted_target.cc"] , "hdrs": ["load_balancing/weighted_target/weighted_target.h"] , "deps": [ "channel_args" , "connectivity_state" , "delegating_helper" , "grpc_lb_address_filtering" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "pollset_set" , "time" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "work_serializer"] ] } , "grpc_lb_policy_xds_override_host": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_lb_policy_xds_override_host"] , "stage": ["src", "core"] , "srcs": ["load_balancing/xds/xds_override_host.cc"] , "hdrs": ["load_balancing/xds/xds_override_host.h"] , "deps": [ "channel_args" , "client_channel_internal_header" , "closure" , "connectivity_state" , "delegating_helper" , "error" , "experiments" , "grpc_stateful_session_filter" , "grpc_xds_client" , "iomgr_fwd" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" , "match" , "pollset_set" , "ref_counted_string" , "resolved_address" , "subchannel_interface" , "validation_errors" , "xds_config" , "xds_health_status" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_trace"] , ["", "lb_child_policy_handler"] , ["", "orphanable"] , ["", "parse_address"] , ["", "ref_counted_ptr"] , ["", "server_address"] , ["", "sockaddr_utils"] , ["", "work_serializer"] ] } , "lb_server_load_reporting_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["lb_server_load_reporting_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/load_reporting/server_load_reporting_filter.cc"] , "hdrs": [ "ext/filters/load_reporting/registered_opencensus_objects.h" , "ext/filters/load_reporting/server_load_reporting_filter.h" , ["src/cpp/server/load_reporter", "constants.h"] ] , "deps": [ "arena_promise" , "call_finalization" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "grpc_sockaddr" , "latent_see" , "metadata_batch" , "resolved_address" , "seq" , "slice" , ["@", "absl", "absl/container", "vector"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "opencensus-stats", "", ""] , ["@", "opencensus-tags", "", ""] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] , ["", "parse_address"] , ["", "promise"] , ["", "uri"] ] } , "grpc_backend_metric_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_backend_metric_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/backend_metrics/backend_metric_filter.cc"] , "hdrs": ["ext/filters/backend_metrics/backend_metric_filter.h"] , "deps": [ "arena_promise" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "experiments" , "grpc_backend_metric_data" , "grpc_backend_metric_provider" , "latent_see" , "map" , "metadata_batch" , "slice" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_trace"] , ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "base"] , ["third_party/upb", "mem"] ] } , "polling_resolver": { "type": ["@", "rules", "CC", "library"] , "name": ["polling_resolver"] , "stage": ["src", "core"] , "srcs": ["resolver/polling_resolver.cc"] , "hdrs": ["resolver/polling_resolver.h"] , "deps": [ "channel_args" , "grpc_service_config" , "iomgr_fwd" , "time" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "backoff"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_resolver"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] , ["", "work_serializer"] ] } , "service_config_helper": { "type": ["@", "rules", "CC", "library"] , "name": ["service_config_helper"] , "stage": ["src", "core"] , "srcs": ["resolver/dns/event_engine/service_config_helper.cc"] , "hdrs": ["resolver/dns/event_engine/service_config_helper.h"] , "deps": [ "json" , "json_args" , "json_object_loader" , "json_reader" , "json_writer" , "status_helper" , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "iomgr"] ] } , "grpc_resolver_dns_event_engine": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_dns_event_engine"] , "stage": ["src", "core"] , "srcs": ["resolver/dns/event_engine/event_engine_client_channel_resolver.cc"] , "hdrs": ["resolver/dns/event_engine/event_engine_client_channel_resolver.h"] , "deps": [ "channel_args" , "event_engine_common" , "grpc_service_config" , "polling_resolver" , "service_config_helper" , "time" , "validation_errors" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "backoff"] , ["", "channel_arg_names"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_grpclb_balancer_addresses"] , ["", "grpc_resolver"] , ["", "grpc_service_config_impl"] , ["", "grpc_trace"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "grpc_resolver_dns_plugin": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_dns_plugin"] , "stage": ["src", "core"] , "srcs": ["resolver/dns/dns_resolver_plugin.cc"] , "hdrs": ["resolver/dns/dns_resolver_plugin.h"] , "deps": [ "experiments" , "grpc_resolver_dns_event_engine" , "grpc_resolver_dns_native" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "config_vars"] , ["", "gpr"] , ["", "grpc_resolver"] , ["", "grpc_resolver_dns_ares"] ] } , "grpc_resolver_dns_native": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_dns_native"] , "stage": ["src", "core"] , "srcs": ["resolver/dns/native/dns_resolver.cc"] , "hdrs": ["resolver/dns/native/dns_resolver.h"] , "deps": [ "channel_args" , "polling_resolver" , "resolved_address" , "time" , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "backoff"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_resolver"] , ["", "grpc_trace"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "grpc_resolver_sockaddr": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_sockaddr"] , "stage": ["src", "core"] , "srcs": ["resolver/sockaddr/sockaddr_resolver.cc"] , "deps": [ "channel_args" , "iomgr_port" , "resolved_address" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_resolver"] , ["", "orphanable"] , ["", "parse_address"] , ["", "uri"] ] } , "grpc_resolver_xds_attributes": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_xds_attributes"] , "stage": ["src", "core"] , "hdrs": ["resolver/xds/xds_resolver_attributes.h"] , "deps": [ "grpc_service_config" , "unique_type_name" , "xds_route_config" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "xds_config": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_config"] , "stage": ["src", "core"] , "srcs": ["resolver/xds/xds_config.cc"] , "hdrs": ["resolver/xds/xds_config.h"] , "deps": [ "match" , "ref_counted" , "xds_cluster" , "xds_endpoint" , "xds_listener" , "xds_route_config" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "variant"] ] } , "xds_dependency_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["xds_dependency_manager"] , "stage": ["src", "core"] , "srcs": ["resolver/xds/xds_dependency_manager.cc"] , "hdrs": ["resolver/xds/xds_dependency_manager.h"] , "deps": [ "grpc_lb_xds_channel_args" , "grpc_xds_client" , "match" , "ref_counted" , "xds_cluster" , "xds_config" , "xds_endpoint" , "xds_listener" , "xds_route_config" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/strings", "strings"] , ["", "config"] , ["", "gpr"] , ["", "grpc_resolver"] , ["", "grpc_resolver_fake"] ] } , "grpc_resolver_xds": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_xds"] , "stage": ["src", "core"] , "srcs": ["resolver/xds/xds_resolver.cc"] , "deps": [ "arena" , "arena_promise" , "channel_args" , "channel_fwd" , "client_channel_internal_header" , "config_selector" , "context" , "dual_ref_counted" , "grpc_lb_policy_ring_hash" , "grpc_resolver_xds_attributes" , "grpc_service_config" , "grpc_xds_client" , "iomgr_fwd" , "match" , "metadata_batch" , "pollset_set" , "ref_counted" , "slice" , "time" , "xds_config" , "xds_dependency_manager" , "xds_http_filter" , "xds_listener" , "xds_route_config" , "xxhash_inline" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["@", "re2", "", "re2"] , ["", "channel_arg_names"] , ["", "config"] , ["", "debug_location"] , ["", "endpoint_addresses"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_resolver"] , ["", "grpc_service_config_impl"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] , ["", "work_serializer"] , ["", "xds_client"] ] } , "grpc_resolver_c2p": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_c2p"] , "stage": ["src", "core"] , "srcs": ["resolver/google_c2p/google_c2p_resolver.cc"] , "deps": [ "channel_args" , "env" , "gcp_metadata_query" , "grpc_xds_client" , "json" , "json_writer" , "resource_quota" , "time" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "alts_util"] , ["", "config"] , ["", "debug_location"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_resolver"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] , ["", "work_serializer"] , ["", "xds_client"] ] } , "hpack_constants": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_constants"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chttp2/transport/hpack_constants.h"] , "deps": [["", "gpr_platform"]] } , "hpack_encoder_table": { "type": ["@", "rules", "CC", "library"] , "name": ["hpack_encoder_table"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/hpack_encoder_table.cc"] , "hdrs": ["ext/transport/chttp2/transport/hpack_encoder_table.h"] , "deps": ["hpack_constants", ["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "chttp2_flow_control": { "type": ["@", "rules", "CC", "library"] , "name": ["chttp2_flow_control"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/flow_control.cc"] , "hdrs": ["ext/transport/chttp2/transport/flow_control.h"] , "deps": [ "bdp_estimator" , "experiments" , "http2_settings" , "memory_quota" , "time" , "useful" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_trace"] ] } , "ping_abuse_policy": { "type": ["@", "rules", "CC", "library"] , "name": ["ping_abuse_policy"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/ping_abuse_policy.cc"] , "hdrs": ["ext/transport/chttp2/transport/ping_abuse_policy.h"] , "deps": [ "channel_args" , "time" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "gpr_platform"] ] } , "ping_callbacks": { "type": ["@", "rules", "CC", "library"] , "name": ["ping_callbacks"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/ping_callbacks.cc"] , "hdrs": ["ext/transport/chttp2/transport/ping_callbacks.h"] , "deps": [ "time" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "distributions"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "write_size_policy": { "type": ["@", "rules", "CC", "library"] , "name": ["write_size_policy"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/write_size_policy.cc"] , "hdrs": ["ext/transport/chttp2/transport/write_size_policy.h"] , "deps": [ "time" , ["@", "absl", "absl/log", "check"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "ping_rate_policy": { "type": ["@", "rules", "CC", "library"] , "name": ["ping_rate_policy"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/ping_rate_policy.cc"] , "hdrs": ["ext/transport/chttp2/transport/ping_rate_policy.h"] , "deps": [ "channel_args" , "experiments" , "match" , "time" , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "channel_arg_names"] , ["", "gpr_platform"] ] } , "huffsyms": { "type": ["@", "rules", "CC", "library"] , "name": ["huffsyms"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/huffsyms.cc"] , "hdrs": ["ext/transport/chttp2/transport/huffsyms.h"] , "deps": [["", "gpr_platform"]] } , "decode_huff": { "type": ["@", "rules", "CC", "library"] , "name": ["decode_huff"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/decode_huff.cc"] , "hdrs": ["ext/transport/chttp2/transport/decode_huff.h"] , "deps": [["", "gpr_platform"]] } , "http2_settings": { "type": ["@", "rules", "CC", "library"] , "name": ["http2_settings"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/http2_settings.cc"] , "hdrs": ["ext/transport/chttp2/transport/http2_settings.h"] , "deps": [ "http2_errors" , "useful" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "chttp2_frame"] , ["", "gpr_platform"] ] } , "grpc_transport_chttp2_alpn": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_chttp2_alpn"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/alpn/alpn.cc"] , "hdrs": ["ext/transport/chttp2/alpn/alpn.h"] , "deps": ["useful", ["@", "absl", "absl/log", "check"], ["", "gpr"]] } , "grpc_transport_chttp2_client_connector": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_chttp2_client_connector"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/client/chttp2_connector.cc"] , "hdrs": ["ext/transport/chttp2/client/chttp2_connector.h"] , "deps": [ "channel_args" , "channel_args_endpoint_config" , "channel_args_preconditioning" , "channel_stack_type" , "closure" , "error" , "error_utils" , "grpc_insecure_credentials" , "handshaker_registry" , "resolved_address" , "status_helper" , "subchannel_connector" , "tcp_connect_handshaker" , "time" , "unique_type_name" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] , ["", "channel"] , ["", "channel_arg_names"] , ["", "channel_create"] , ["", "channelz"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_public_hdrs"] , ["", "grpc_resolver"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "grpc_transport_chttp2"] , ["", "handshaker"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] ] } , "grpc_transport_chttp2_server": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_chttp2_server"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/server/chttp2_server.cc"] , "hdrs": ["ext/transport/chttp2/server/chttp2_server.h"] , "deps": [ "channel_args" , "channel_args_endpoint_config" , "closure" , "connection_quota" , "error" , "error_utils" , "event_engine_common" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "event_engine_utils" , "grpc_insecure_credentials" , "handshaker_registry" , "iomgr_fwd" , "match" , "memory_quota" , "pollset_set" , "resolved_address" , "resource_quota" , "status_helper" , "time" , "unique_type_name" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel_arg_names"] , ["", "channelz"] , ["", "chttp2_legacy_frame"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "grpc_transport_chttp2"] , ["", "handshaker"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server"] , ["", "sockaddr_utils"] , ["", "uri"] ] } , "grpc_transport_inproc": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_inproc"] , "stage": ["src", "core"] , "srcs": [ "ext/transport/inproc/inproc_transport.cc" , "ext/transport/inproc/legacy_inproc_transport.cc" ] , "hdrs": [ "ext/transport/inproc/inproc_transport.h" , "ext/transport/inproc/legacy_inproc_transport.h" ] , "deps": [ "arena" , "channel_args" , "channel_args_preconditioning" , "channel_stack_type" , "closure" , "connectivity_state" , "error" , "event_engine_context" , "experiments" , "iomgr_fwd" , "metadata" , "metadata_batch" , "slice" , "slice_buffer" , "status_helper" , "time" , "try_seq" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "channel"] , ["", "channel_arg_names"] , ["", "channel_create"] , ["", "channelz"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] , ["", "iomgr"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["", "server"] ] } , "chaotic_good_frame_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["chaotic_good_frame_proto"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/chaotic_good_frame_proto.proto"] } , "chaotic_good_frame_cc_proto": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_frame_cc_proto"] , "stage": ["src", "core"] , "deps": ["chaotic_good_frame_proto"] } , "chaotic_good_frame": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_frame"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/frame.cc"] , "hdrs": ["ext/transport/chaotic_good/frame.h"] , "deps": [ "arena" , "bitset" , "chaotic_good_frame_cc_proto" , "chaotic_good_frame_header" , "context" , "match" , "message" , "metadata" , "metadata_batch" , "no_destruct" , "slice" , "slice_buffer" , "status_helper" , ["", "gpr_platform"] , ["", "grpc_base"] ] } , "chaotic_good_frame_header": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_frame_header"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/frame_header.cc"] , "hdrs": ["ext/transport/chaotic_good/frame_header.h"] , "deps": [ "bitset" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "chaotic_good_legacy_frame": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_frame"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/frame.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/frame.h"] , "deps": [ "arena" , "bitset" , "chaotic_good_legacy_frame_header" , "context" , "match" , "metadata_batch" , "no_destruct" , "slice" , "slice_buffer" , "status_helper" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "variant"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "hpack_encoder"] , ["", "hpack_parser"] ] } , "chaotic_good_legacy_settings_metadata": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_settings_metadata"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/settings_metadata.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/settings_metadata.h"] , "deps": [ "arena" , "metadata_batch" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] ] } , "chaotic_good_legacy_frame_header": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_frame_header"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/frame_header.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/frame_header.h"] , "deps": [ "bitset" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "gpr_platform"] ] } , "gcp_metadata_query": { "type": ["@", "rules", "CC", "library"] , "name": ["gcp_metadata_query"] , "stage": ["src", "core"] , "srcs": ["util/gcp_metadata_query.cc"] , "hdrs": ["util/gcp_metadata_query.h"] , "deps": [ "closure" , "error" , "status_helper" , "time" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_core_credentials_header"] , ["", "grpc_security_base"] , ["", "grpc_trace"] , ["", "httpcli"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri"] ] } , "logging_sink": { "type": ["@", "rules", "CC", "library"] , "name": ["logging_sink"] , "stage": ["src", "core"] , "hdrs": ["ext/filters/logging/logging_sink.h"] , "deps": [ "time" , ["@", "absl", "absl/numeric", "int128"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "logging_filter": { "type": ["@", "rules", "CC", "library"] , "name": ["logging_filter"] , "stage": ["src", "core"] , "srcs": ["ext/filters/logging/logging_filter.cc"] , "hdrs": ["ext/filters/logging/logging_filter.h"] , "deps": [ "arena" , "arena_promise" , "cancel_callback" , "channel_args" , "channel_fwd" , "channel_stack_type" , "context" , "latent_see" , "logging_sink" , "map" , "metadata_batch" , "pipe" , "poll" , "slice" , "slice_buffer" , "time" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/numeric", "int128"] , ["@", "absl", "absl/random", "distributions"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "call_tracer"] , ["", "channel_arg_names"] , ["", "config"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "grpc_public_hdrs"] , ["", "grpc_resolver"] , ["", "uri"] ] } , "grpc_promise_endpoint": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_promise_endpoint"] , "stage": ["src", "core"] , "srcs": ["lib/transport/promise_endpoint.cc"] , "hdrs": ["lib/transport/promise_endpoint.h"] , "deps": [ "activity" , "cancel_callback" , "event_engine_common" , "event_engine_extensions" , "event_engine_query_extensions" , "if" , "map" , "poll" , "slice" , "slice_buffer" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] , ["", "gpr"] ] } , "chaotic_good_config": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_config"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good/config.h"] , "deps": [ "channel_args" , "chaotic_good_frame_cc_proto" , "chaotic_good_message_chunker" , "chaotic_good_pending_connection" , "chaotic_good_transport" , "event_engine_extensions" , ["@", "absl", "absl/container", "flat_hash_set"] ] } , "chaotic_good_message_chunker": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_message_chunker"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good/message_chunker.h"] , "deps": ["chaotic_good_frame", "if", "loop", "map", "seq"] } , "chaotic_good_message_reassembly": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_message_reassembly"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good/message_reassembly.h"] , "deps": ["call_spine", "chaotic_good_frame", ["@", "absl", "absl/log", "log"]] } , "chaotic_good_control_endpoint": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_control_endpoint"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/control_endpoint.cc"] , "hdrs": ["ext/transport/chaotic_good/control_endpoint.h"] , "deps": [ "1999" , "event_engine_context" , "event_engine_tcp_socket_utils" , "grpc_promise_endpoint" , "loop" , "try_seq" , ["@", "absl", "absl/cleanup", "cleanup"] , ["", "gpr"] ] } , "chaotic_good_pending_connection": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_pending_connection"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good/pending_connection.h"] , "deps": [ "dual_ref_counted" , "grpc_promise_endpoint" , ["@", "absl", "absl/status", "statusor"] , ["", "promise"] ] } , "chaotic_good_data_endpoints": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_data_endpoints"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/data_endpoints.cc"] , "hdrs": ["ext/transport/chaotic_good/data_endpoints.h"] , "deps": [ "1999" , "chaotic_good_pending_connection" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "grpc_promise_endpoint" , "loop" , "seq" , "slice_buffer" , "try_seq" , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/strings", "strings"] , ["", "promise"] ] } , "chaotic_good_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_transport"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good/chaotic_good_transport.h"] , "deps": [ "call_spine" , "chaotic_good_control_endpoint" , "chaotic_good_data_endpoint" , "chaotic_good_frame" , "chaotic_good_frame_header" , "chaotic_good_pending_connection" , "event_engine_context" , "event_engine_tcp_socket_utils" , "grpc_promise_endpoint" , "loop" , "match_promise" , "mpsc" , "seq" , "try_join" , "try_seq" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "chaotic_good_client_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_client_transport"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/chaotic_good_client_transport.cc"] , "hdrs": ["ext/transport/chaotic_good/chaotic_good_client_transport.h"] , "deps": [ "activity" , "arena" , "chaotic_good_config" , "chaotic_good_frame" , "chaotic_good_frame_header" , "chaotic_good_message_reassembly" , "chaotic_good_pending_connection" , "chaotic_good_transport" , "context" , "event_engine_context" , "event_engine_query_extensions" , "for_each" , "grpc_promise_endpoint" , "if" , "inter_activity_pipe" , "loop" , "map" , "memory_quota" , "metadata_batch" , "mpsc" , "pipe" , "poll" , "resource_quota" , "slice" , "slice_buffer" , "switch" , "try_join" , "try_seq" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] ] } , "chaotic_good_server_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_server_transport"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/chaotic_good_server_transport.cc"] , "hdrs": ["ext/transport/chaotic_good/chaotic_good_server_transport.h"] , "deps": [ "1999" , "activity" , "arena" , "chaotic_good_frame" , "chaotic_good_frame_header" , "chaotic_good_transport" , "context" , "default_event_engine" , "event_engine_context" , "event_engine_wakeup_scheduler" , "for_each" , "grpc_promise_endpoint" , "if" , "inter_activity_latch" , "inter_activity_pipe" , "loop" , "memory_quota" , "metadata_batch" , "mpsc" , "pipe" , "poll" , "resource_quota" , "seq" , "slice" , "slice_buffer" , "switch" , "try_join" , "try_seq" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "ref_counted_ptr"] ] } , "chaotic_good_legacy_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_transport"] , "stage": ["src", "core"] , "hdrs": ["ext/transport/chaotic_good_legacy/chaotic_good_server_transport.h"] , "deps": [ "chaotic_good_legacy_frame" , "chaotic_good_legacy_frame_header" , "event_engine_tcp_socket_utils" , "grpc_promise_endpoint" , "if" , "try_join" , "try_seq" , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "random"] , ["", "gpr_platform"] , ["", "grpc_trace"] , ["", "hpack_encoder"] , ["", "promise"] ] } , "chaotic_good_legacy_client_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_client_transport"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/client_transport.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/client_transport.h"] , "deps": [ "activity" , "arena" , "chaotic_good_legacy_frame" , "chaotic_good_legacy_frame_header" , "chaotic_good_legacy_transport" , "context" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "for_each" , "grpc_promise_endpoint" , "if" , "inter_activity_pipe" , "loop" , "map" , "memory_quota" , "metadata_batch" , "mpsc" , "pipe" , "poll" , "resource_quota" , "slice" , "slice_buffer" , "try_join" , "try_seq" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "hpack_encoder"] , ["", "hpack_parser"] , ["", "ref_counted_ptr"] ] } , "chaotic_good_legacy_server_transport": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_server_transport"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/server_transport.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/server_transport.h"] , "deps": [ "1999" , "activity" , "arena" , "chaotic_good_legacy_frame" , "chaotic_good_legacy_frame_header" , "chaotic_good_legacy_transport" , "context" , "default_event_engine" , "event_engine_context" , "event_engine_wakeup_scheduler" , "for_each" , "grpc_promise_endpoint" , "if" , "inter_activity_latch" , "inter_activity_pipe" , "loop" , "memory_quota" , "metadata_batch" , "mpsc" , "pipe" , "poll" , "resource_quota" , "seq" , "slice" , "slice_buffer" , "switch" , "try_join" , "try_seq" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "hpack_encoder"] , ["", "hpack_parser"] , ["", "ref_counted_ptr"] ] } , "call_final_info": { "type": ["@", "rules", "CC", "library"] , "name": ["call_final_info"] , "stage": ["src", "core"] , "srcs": ["lib/transport/call_final_info.cc"] , "hdrs": ["lib/transport/call_final_info.h"] , "deps": [["", "gpr"], ["", "grpc_public_hdrs"]] } , "call_finalization": { "type": ["@", "rules", "CC", "library"] , "name": ["call_finalization"] , "stage": ["src", "core"] , "hdrs": ["lib/channel/call_finalization.h"] , "deps": ["arena", "call_final_info", "context", ["", "gpr_platform"]] } , "call_state": { "type": ["@", "rules", "CC", "library"] , "name": ["call_state"] , "stage": ["src", "core"] , "srcs": ["lib/transport/call_state.cc"] , "hdrs": ["lib/transport/call_state.h"] , "deps": ["activity", "poll", "status_flag", ["", "gpr"], ["", "grpc_trace"]] } , "call_filters": { "type": ["@", "rules", "CC", "library"] , "name": ["call_filters"] , "stage": ["src", "core"] , "srcs": ["lib/transport/call_filters.cc"] , "hdrs": ["lib/transport/call_filters.h"] , "deps": [ "call_final_info" , "call_state" , "dump_args" , "for_each" , "if" , "latch" , "map" , "message" , "metadata" , "ref_counted" , "seq" , "status_flag" , "try_seq" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["", "gpr"] , ["", "promise"] , ["", "ref_counted_ptr"] ] } , "interception_chain": { "type": ["@", "rules", "CC", "library"] , "name": ["interception_chain"] , "stage": ["src", "core"] , "srcs": ["lib/transport/interception_chain.cc"] , "hdrs": ["lib/transport/interception_chain.h"] , "deps": [ "call_destination" , "call_filters" , "call_spine" , "match" , "metadata" , "ref_counted" , ["", "gpr_platform"] , ["", "grpc_trace"] ] } , "call_destination": { "type": ["@", "rules", "CC", "library"] , "name": ["call_destination"] , "stage": ["src", "core"] , "hdrs": ["lib/transport/call_destination.h"] , "deps": ["call_spine", ["", "gpr_platform"], ["", "orphanable"]] } , "parsed_metadata": { "type": ["@", "rules", "CC", "library"] , "name": ["parsed_metadata"] , "stage": ["src", "core"] , "srcs": ["lib/transport/parsed_metadata.cc"] , "hdrs": ["lib/transport/parsed_metadata.h"] , "deps": [ "slice" , "time" , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] ] } , "metadata": { "type": ["@", "rules", "CC", "library"] , "name": ["metadata"] , "stage": ["src", "core"] , "srcs": ["lib/transport/metadata.cc"] , "hdrs": ["lib/transport/metadata.h"] , "deps": ["error_utils", "metadata_batch", ["", "gpr_platform"]] } , "message": { "type": ["@", "rules", "CC", "library"] , "name": ["message"] , "stage": ["src", "core"] , "srcs": ["lib/transport/message.cc"] , "hdrs": ["lib/transport/message.h"] , "deps": [ "arena" , "slice_buffer" , ["@", "absl", "absl/strings", "strings"] , ["", "gpr_platform"] , ["", "grpc_public_hdrs"] ] } , "call_spine": { "type": ["@", "rules", "CC", "library"] , "name": ["call_spine"] , "stage": ["src", "core"] , "srcs": ["lib/transport/call_spine.cc"] , "hdrs": ["lib/transport/call_spine.h"] , "deps": [ "1999" , "call_arena_allocator" , "call_filters" , "dual_ref_counted" , "event_engine_context" , "for_each" , "if" , "latch" , "message" , "metadata" , "pipe" , "prioritized_race" , "promise_status" , "status_flag" , "try_seq" , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/log", "check"] , ["", "gpr"] , ["", "promise"] ] } , "direct_channel": { "type": ["@", "rules", "CC", "library"] , "name": ["direct_channel"] , "stage": ["src", "core"] , "srcs": ["client_channel/direct_channel.cc"] , "hdrs": ["client_channel/direct_channel.h"] , "deps": [ "channel_stack_type" , "event_engine_context" , "interception_chain" , ["", "channel"] , ["", "config"] , ["", "grpc_base"] , ["", "orphanable"] ] } , "metadata_batch": { "type": ["@", "rules", "CC", "library"] , "name": ["metadata_batch"] , "stage": ["src", "core"] , "srcs": ["lib/transport/metadata_batch.cc"] , "hdrs": [ "lib/transport/custom_metadata.h" , "lib/transport/metadata_batch.h" , "lib/transport/simple_slice_based_metadata.h" ] , "deps": [ "arena" , "chunked_vector" , "compression" , "experiments" , "if_list" , "metadata_compression_traits" , "packed_table" , "parsed_metadata" , "poll" , "slice" , "time" , "timeout_encoding" , "type_list" , ["@", "absl", "absl/base", "no_destructor"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_public_hdrs"] ] } , "timeout_encoding": { "type": ["@", "rules", "CC", "library"] , "name": ["timeout_encoding"] , "stage": ["src", "core"] , "srcs": ["lib/transport/timeout_encoding.cc"] , "hdrs": ["lib/transport/timeout_encoding.h"] , "deps": [ "slice" , "time" , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] ] } , "call_arena_allocator": { "type": ["@", "rules", "CC", "library"] , "name": ["call_arena_allocator"] , "stage": ["src", "core"] , "srcs": ["lib/transport/call_arena_allocator.cc"] , "hdrs": ["lib/transport/call_arena_allocator.h"] , "deps": ["arena", "memory_quota", "ref_counted", ["", "gpr_platform"]] } , "compression": { "type": ["@", "rules", "CC", "library"] , "name": ["compression"] , "stage": ["src", "core"] , "srcs": [ "lib/compression/compression.cc" , "lib/compression/compression_internal.cc" ] , "hdrs": ["lib/compression/compression_internal.h"] , "deps": [ "bitset" , "channel_args" , "ref_counted_string" , "slice" , "useful" , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["", "gpr"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] ] } , "chaotic_good_server": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_server"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/server/chaotic_good_server.cc"] , "hdrs": ["ext/transport/chaotic_good/server/chaotic_good_server.h"] , "deps": [ "activity" , "arena" , "channel_args" , "channel_args_endpoint_config" , "chaotic_good_config" , "chaotic_good_frame" , "chaotic_good_frame_header" , "chaotic_good_legacy_server" , "chaotic_good_pending_connection" , "chaotic_good_server_transport" , "chaotic_good_transport" , "closure" , "context" , "error" , "error_utils" , "event_engine_common" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "event_engine_utils" , "event_engine_wakeup_scheduler" , "grpc_promise_endpoint" , "if" , "inter_activity_latch" , "iomgr_fwd" , "join" , "latch" , "memory_quota" , "metadata" , "metadata_batch" , "race" , "resource_quota" , "sleep" , "slice" , "slice_buffer" , "status_helper" , "time" , "try_seq" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "channelz"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "handshaker"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server"] ] } , "chaotic_good_legacy_server": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_server"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/server/chaotic_good_server.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/server/chaotic_good_server.h"] , "deps": [ "activity" , "arena" , "channel_args" , "channel_args_endpoint_config" , "chaotic_good_legacy_frame" , "chaotic_good_legacy_frame_header" , "chaotic_good_legacy_server_transport" , "chaotic_good_legacy_settings_metadata" , "closure" , "context" , "error" , "error_utils" , "event_engine_common" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "event_engine_wakeup_scheduler" , "grpc_promise_endpoint" , "if" , "inter_activity_latch" , "iomgr_fwd" , "latch" , "memory_quota" , "metadata" , "metadata_batch" , "race" , "resource_quota" , "sleep" , "slice" , "slice_buffer" , "status_helper" , "time" , "try_seq" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "channelz"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "handshaker"] , ["", "hpack_encoder"] , ["", "hpack_parser"] , ["", "iomgr"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server"] ] } , "chaotic_good_connector": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_connector"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good/client/chaotic_good_connector.cc"] , "hdrs": ["ext/transport/chaotic_good/client/chaotic_good_connector.h"] , "deps": [ "activity" , "all_ok" , "arena" , "channel_args" , "channel_args_endpoint_config" , "chaotic_good_client_transport" , "chaotic_good_config" , "chaotic_good_frame" , "chaotic_good_frame_cc_proto" , "chaotic_good_frame_header" , "chaotic_good_legacy_connector" , "chaotic_good_transport" , "closure" , "context" , "error" , "error_utils" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "event_engine_wakeup_scheduler" , "grpc_promise_endpoint" , "inter_activity_latch" , "latch" , "memory_quota" , "no_destruct" , "notification" , "race" , "resource_quota" , "sleep" , "slice" , "slice_buffer" , "subchannel_connector" , "time" , "try_seq" , "wait_for_callback" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "channel"] , ["", "channel_create"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "handshaker"] , ["", "iomgr"] , ["", "ref_counted_ptr"] ] } , "chaotic_good_legacy_connector": { "type": ["@", "rules", "CC", "library"] , "name": ["chaotic_good_legacy_connector"] , "stage": ["src", "core"] , "srcs": ["ext/transport/chaotic_good_legacy/client/chaotic_good_connector.cc"] , "hdrs": ["ext/transport/chaotic_good_legacy/client/chaotic_good_connector.h"] , "deps": [ "activity" , "arena" , "channel_args" , "channel_args_endpoint_config" , "chaotic_good_legacy_client_transport" , "chaotic_good_legacy_frame" , "chaotic_good_legacy_frame_header" , "chaotic_good_legacy_settings_metadata" , "closure" , "context" , "error" , "error_utils" , "event_engine_context" , "event_engine_extensions" , "event_engine_query_extensions" , "event_engine_tcp_socket_utils" , "event_engine_wakeup_scheduler" , "grpc_promise_endpoint" , "inter_activity_latch" , "latch" , "memory_quota" , "no_destruct" , "notification" , "race" , "resource_quota" , "sleep" , "slice" , "slice_buffer" , "subchannel_connector" , "time" , "try_seq" , "wait_for_callback" , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/random", "bit_gen_ref"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["", "channel"] , ["", "channel_create"] , ["", "config"] , ["", "debug_location"] , ["", "exec_ctx"] , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_client_channel"] , ["", "handshaker"] , ["", "hpack_encoder"] , ["", "hpack_parser"] , ["", "iomgr"] , ["", "ref_counted_ptr"] ] } , "metrics": { "type": ["@", "rules", "CC", "library"] , "name": ["metrics"] , "stage": ["src", "core"] , "srcs": ["telemetry/metrics.cc"] , "hdrs": ["telemetry/metrics.h"] , "deps": [ "arena" , "channel_args" , "no_destruct" , "slice" , "time" , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/log", "check"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["", "call_tracer"] , ["", "gpr"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/ext/000077500000000000000000000000001516554100600242235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/ext/upb-gen/000077500000000000000000000000001516554100600255605ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/ext/upb-gen/TARGETS.grpc000066400000000000000000000651141516554100600275550ustar00rootroot00000000000000{ "upb-gen-lib": { "type": ["@", "rules", "CC", "library"] , "name": ["upb-gen-lib"] , "pure C": ["YES"] , "srcs": [ "envoy/admin/v3/certs.upb_minitable.c" , "envoy/admin/v3/clusters.upb_minitable.c" , "envoy/admin/v3/config_dump.upb_minitable.c" , "envoy/admin/v3/config_dump_shared.upb_minitable.c" , "envoy/admin/v3/init_dump.upb_minitable.c" , "envoy/admin/v3/listeners.upb_minitable.c" , "envoy/admin/v3/memory.upb_minitable.c" , "envoy/admin/v3/metrics.upb_minitable.c" , "envoy/admin/v3/mutex_stats.upb_minitable.c" , "envoy/admin/v3/server_info.upb_minitable.c" , "envoy/admin/v3/tap.upb_minitable.c" , "envoy/annotations/deprecation.upb_minitable.c" , "envoy/annotations/resource.upb_minitable.c" , "envoy/config/accesslog/v3/accesslog.upb_minitable.c" , "envoy/config/bootstrap/v3/bootstrap.upb_minitable.c" , "envoy/config/cluster/v3/circuit_breaker.upb_minitable.c" , "envoy/config/cluster/v3/cluster.upb_minitable.c" , "envoy/config/cluster/v3/filter.upb_minitable.c" , "envoy/config/cluster/v3/outlier_detection.upb_minitable.c" , "envoy/config/common/matcher/v3/matcher.upb_minitable.c" , "envoy/config/core/v3/address.upb_minitable.c" , "envoy/config/core/v3/backoff.upb_minitable.c" , "envoy/config/core/v3/base.upb_minitable.c" , "envoy/config/core/v3/config_source.upb_minitable.c" , "envoy/config/core/v3/event_service_config.upb_minitable.c" , "envoy/config/core/v3/extension.upb_minitable.c" , "envoy/config/core/v3/grpc_method_list.upb_minitable.c" , "envoy/config/core/v3/grpc_service.upb_minitable.c" , "envoy/config/core/v3/health_check.upb_minitable.c" , "envoy/config/core/v3/http_service.upb_minitable.c" , "envoy/config/core/v3/http_uri.upb_minitable.c" , "envoy/config/core/v3/protocol.upb_minitable.c" , "envoy/config/core/v3/proxy_protocol.upb_minitable.c" , "envoy/config/core/v3/resolver.upb_minitable.c" , "envoy/config/core/v3/socket_cmsg_headers.upb_minitable.c" , "envoy/config/core/v3/socket_option.upb_minitable.c" , "envoy/config/core/v3/substitution_format_string.upb_minitable.c" , "envoy/config/core/v3/udp_socket_config.upb_minitable.c" , "envoy/config/endpoint/v3/endpoint.upb_minitable.c" , "envoy/config/endpoint/v3/endpoint_components.upb_minitable.c" , "envoy/config/endpoint/v3/load_report.upb_minitable.c" , "envoy/config/listener/v3/api_listener.upb_minitable.c" , "envoy/config/listener/v3/listener.upb_minitable.c" , "envoy/config/listener/v3/listener_components.upb_minitable.c" , "envoy/config/listener/v3/quic_config.upb_minitable.c" , "envoy/config/listener/v3/udp_listener_config.upb_minitable.c" , "envoy/config/metrics/v3/metrics_service.upb_minitable.c" , "envoy/config/metrics/v3/stats.upb_minitable.c" , "envoy/config/overload/v3/overload.upb_minitable.c" , "envoy/config/rbac/v3/rbac.upb_minitable.c" , "envoy/config/route/v3/route.upb_minitable.c" , "envoy/config/route/v3/route_components.upb_minitable.c" , "envoy/config/route/v3/scoped_route.upb_minitable.c" , "envoy/config/tap/v3/common.upb_minitable.c" , "envoy/config/trace/v3/datadog.upb_minitable.c" , "envoy/config/trace/v3/dynamic_ot.upb_minitable.c" , "envoy/config/trace/v3/http_tracer.upb_minitable.c" , "envoy/config/trace/v3/lightstep.upb_minitable.c" , "envoy/config/trace/v3/opentelemetry.upb_minitable.c" , "envoy/config/trace/v3/service.upb_minitable.c" , "envoy/config/trace/v3/skywalking.upb_minitable.c" , "envoy/config/trace/v3/trace.upb_minitable.c" , "envoy/config/trace/v3/xray.upb_minitable.c" , "envoy/config/trace/v3/zipkin.upb_minitable.c" , "envoy/data/accesslog/v3/accesslog.upb_minitable.c" , "envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.c" , "envoy/extensions/filters/common/fault/v3/fault.upb_minitable.c" , "envoy/extensions/filters/http/fault/v3/fault.upb_minitable.c" , "envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.c" , "envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.c" , "envoy/extensions/filters/http/router/v3/router.upb_minitable.c" , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.c" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.c" , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.c" , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.c" , "envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.c" , "envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.c" , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.c" , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.c" , "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.c" , "envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.c" , "envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.c" , "envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.c" , "envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.c" , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.c" , "envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.c" , "envoy/service/discovery/v3/ads.upb_minitable.c" , "envoy/service/discovery/v3/discovery.upb_minitable.c" , "envoy/service/load_stats/v3/lrs.upb_minitable.c" , "envoy/service/status/v3/csds.upb_minitable.c" , "envoy/type/http/v3/cookie.upb_minitable.c" , "envoy/type/http/v3/path_transformation.upb_minitable.c" , "envoy/type/matcher/v3/filter_state.upb_minitable.c" , "envoy/type/matcher/v3/http_inputs.upb_minitable.c" , "envoy/type/matcher/v3/metadata.upb_minitable.c" , "envoy/type/matcher/v3/node.upb_minitable.c" , "envoy/type/matcher/v3/number.upb_minitable.c" , "envoy/type/matcher/v3/path.upb_minitable.c" , "envoy/type/matcher/v3/regex.upb_minitable.c" , "envoy/type/matcher/v3/status_code_input.upb_minitable.c" , "envoy/type/matcher/v3/string.upb_minitable.c" , "envoy/type/matcher/v3/struct.upb_minitable.c" , "envoy/type/matcher/v3/value.upb_minitable.c" , "envoy/type/metadata/v3/metadata.upb_minitable.c" , "envoy/type/tracing/v3/custom_tag.upb_minitable.c" , "envoy/type/v3/hash_policy.upb_minitable.c" , "envoy/type/v3/http.upb_minitable.c" , "envoy/type/v3/http_status.upb_minitable.c" , "envoy/type/v3/percent.upb_minitable.c" , "envoy/type/v3/range.upb_minitable.c" , "envoy/type/v3/ratelimit_strategy.upb_minitable.c" , "envoy/type/v3/ratelimit_unit.upb_minitable.c" , "envoy/type/v3/semantic_version.upb_minitable.c" , "envoy/type/v3/token_bucket.upb_minitable.c" , "google/api/annotations.upb_minitable.c" , "google/api/expr/v1alpha1/checked.upb_minitable.c" , "google/api/expr/v1alpha1/syntax.upb_minitable.c" , "google/api/http.upb_minitable.c" , "google/api/httpbody.upb_minitable.c" , "google/protobuf/any.upb_minitable.c" , "google/protobuf/descriptor.upb_minitable.c" , "google/protobuf/duration.upb_minitable.c" , "google/protobuf/empty.upb_minitable.c" , "google/protobuf/struct.upb_minitable.c" , "google/protobuf/timestamp.upb_minitable.c" , "google/protobuf/wrappers.upb_minitable.c" , "google/rpc/status.upb_minitable.c" , "src/proto/grpc/gcp/altscontext.upb_minitable.c" , "src/proto/grpc/gcp/handshaker.upb_minitable.c" , "src/proto/grpc/gcp/transport_security_common.upb_minitable.c" , "src/proto/grpc/health/v1/health.upb_minitable.c" , "src/proto/grpc/lb/v1/load_balancer.upb_minitable.c" , "src/proto/grpc/lookup/v1/rls.upb_minitable.c" , "src/proto/grpc/lookup/v1/rls_config.upb_minitable.c" , "udpa/annotations/migrate.upb_minitable.c" , "udpa/annotations/security.upb_minitable.c" , "udpa/annotations/sensitive.upb_minitable.c" , "udpa/annotations/status.upb_minitable.c" , "udpa/annotations/versioning.upb_minitable.c" , "validate/validate.upb_minitable.c" , "xds/annotations/v3/migrate.upb_minitable.c" , "xds/annotations/v3/security.upb_minitable.c" , "xds/annotations/v3/sensitive.upb_minitable.c" , "xds/annotations/v3/status.upb_minitable.c" , "xds/annotations/v3/versioning.upb_minitable.c" , "xds/core/v3/authority.upb_minitable.c" , "xds/core/v3/cidr.upb_minitable.c" , "xds/core/v3/collection_entry.upb_minitable.c" , "xds/core/v3/context_params.upb_minitable.c" , "xds/core/v3/extension.upb_minitable.c" , "xds/core/v3/resource.upb_minitable.c" , "xds/core/v3/resource_locator.upb_minitable.c" , "xds/core/v3/resource_name.upb_minitable.c" , "xds/data/orca/v3/orca_load_report.upb_minitable.c" , "xds/service/orca/v3/orca.upb_minitable.c" , "xds/type/matcher/v3/cel.upb_minitable.c" , "xds/type/matcher/v3/domain.upb_minitable.c" , "xds/type/matcher/v3/http_inputs.upb_minitable.c" , "xds/type/matcher/v3/ip.upb_minitable.c" , "xds/type/matcher/v3/matcher.upb_minitable.c" , "xds/type/matcher/v3/range.upb_minitable.c" , "xds/type/matcher/v3/regex.upb_minitable.c" , "xds/type/matcher/v3/string.upb_minitable.c" , "xds/type/v3/cel.upb_minitable.c" , "xds/type/v3/range.upb_minitable.c" , "xds/type/v3/typed_struct.upb_minitable.c" ] , "hdrs": [ "envoy/admin/v3/certs.upb.h" , "envoy/admin/v3/certs.upb_minitable.h" , "envoy/admin/v3/clusters.upb.h" , "envoy/admin/v3/clusters.upb_minitable.h" , "envoy/admin/v3/config_dump.upb.h" , "envoy/admin/v3/config_dump.upb_minitable.h" , "envoy/admin/v3/config_dump_shared.upb.h" , "envoy/admin/v3/config_dump_shared.upb_minitable.h" , "envoy/admin/v3/init_dump.upb.h" , "envoy/admin/v3/init_dump.upb_minitable.h" , "envoy/admin/v3/listeners.upb.h" , "envoy/admin/v3/listeners.upb_minitable.h" , "envoy/admin/v3/memory.upb.h" , "envoy/admin/v3/memory.upb_minitable.h" , "envoy/admin/v3/metrics.upb.h" , "envoy/admin/v3/metrics.upb_minitable.h" , "envoy/admin/v3/mutex_stats.upb.h" , "envoy/admin/v3/mutex_stats.upb_minitable.h" , "envoy/admin/v3/server_info.upb.h" , "envoy/admin/v3/server_info.upb_minitable.h" , "envoy/admin/v3/tap.upb.h" , "envoy/admin/v3/tap.upb_minitable.h" , "envoy/annotations/deprecation.upb.h" , "envoy/annotations/deprecation.upb_minitable.h" , "envoy/annotations/resource.upb.h" , "envoy/annotations/resource.upb_minitable.h" , "envoy/config/accesslog/v3/accesslog.upb.h" , "envoy/config/accesslog/v3/accesslog.upb_minitable.h" , "envoy/config/bootstrap/v3/bootstrap.upb.h" , "envoy/config/bootstrap/v3/bootstrap.upb_minitable.h" , "envoy/config/cluster/v3/circuit_breaker.upb.h" , "envoy/config/cluster/v3/circuit_breaker.upb_minitable.h" , "envoy/config/cluster/v3/cluster.upb.h" , "envoy/config/cluster/v3/cluster.upb_minitable.h" , "envoy/config/cluster/v3/filter.upb.h" , "envoy/config/cluster/v3/filter.upb_minitable.h" , "envoy/config/cluster/v3/outlier_detection.upb.h" , "envoy/config/cluster/v3/outlier_detection.upb_minitable.h" , "envoy/config/common/matcher/v3/matcher.upb.h" , "envoy/config/common/matcher/v3/matcher.upb_minitable.h" , "envoy/config/core/v3/address.upb.h" , "envoy/config/core/v3/address.upb_minitable.h" , "envoy/config/core/v3/backoff.upb.h" , "envoy/config/core/v3/backoff.upb_minitable.h" , "envoy/config/core/v3/base.upb.h" , "envoy/config/core/v3/base.upb_minitable.h" , "envoy/config/core/v3/config_source.upb.h" , "envoy/config/core/v3/config_source.upb_minitable.h" , "envoy/config/core/v3/event_service_config.upb.h" , "envoy/config/core/v3/event_service_config.upb_minitable.h" , "envoy/config/core/v3/extension.upb.h" , "envoy/config/core/v3/extension.upb_minitable.h" , "envoy/config/core/v3/grpc_method_list.upb.h" , "envoy/config/core/v3/grpc_method_list.upb_minitable.h" , "envoy/config/core/v3/grpc_service.upb.h" , "envoy/config/core/v3/grpc_service.upb_minitable.h" , "envoy/config/core/v3/health_check.upb.h" , "envoy/config/core/v3/health_check.upb_minitable.h" , "envoy/config/core/v3/http_service.upb.h" , "envoy/config/core/v3/http_service.upb_minitable.h" , "envoy/config/core/v3/http_uri.upb.h" , "envoy/config/core/v3/http_uri.upb_minitable.h" , "envoy/config/core/v3/protocol.upb.h" , "envoy/config/core/v3/protocol.upb_minitable.h" , "envoy/config/core/v3/proxy_protocol.upb.h" , "envoy/config/core/v3/proxy_protocol.upb_minitable.h" , "envoy/config/core/v3/resolver.upb.h" , "envoy/config/core/v3/resolver.upb_minitable.h" , "envoy/config/core/v3/socket_cmsg_headers.upb.h" , "envoy/config/core/v3/socket_cmsg_headers.upb_minitable.h" , "envoy/config/core/v3/socket_option.upb.h" , "envoy/config/core/v3/socket_option.upb_minitable.h" , "envoy/config/core/v3/substitution_format_string.upb.h" , "envoy/config/core/v3/substitution_format_string.upb_minitable.h" , "envoy/config/core/v3/udp_socket_config.upb.h" , "envoy/config/core/v3/udp_socket_config.upb_minitable.h" , "envoy/config/endpoint/v3/endpoint.upb.h" , "envoy/config/endpoint/v3/endpoint.upb_minitable.h" , "envoy/config/endpoint/v3/endpoint_components.upb.h" , "envoy/config/endpoint/v3/endpoint_components.upb_minitable.h" , "envoy/config/endpoint/v3/load_report.upb.h" , "envoy/config/endpoint/v3/load_report.upb_minitable.h" , "envoy/config/listener/v3/api_listener.upb.h" , "envoy/config/listener/v3/api_listener.upb_minitable.h" , "envoy/config/listener/v3/listener.upb.h" , "envoy/config/listener/v3/listener.upb_minitable.h" , "envoy/config/listener/v3/listener_components.upb.h" , "envoy/config/listener/v3/listener_components.upb_minitable.h" , "envoy/config/listener/v3/quic_config.upb.h" , "envoy/config/listener/v3/quic_config.upb_minitable.h" , "envoy/config/listener/v3/udp_listener_config.upb.h" , "envoy/config/listener/v3/udp_listener_config.upb_minitable.h" , "envoy/config/metrics/v3/metrics_service.upb.h" , "envoy/config/metrics/v3/metrics_service.upb_minitable.h" , "envoy/config/metrics/v3/stats.upb.h" , "envoy/config/metrics/v3/stats.upb_minitable.h" , "envoy/config/overload/v3/overload.upb.h" , "envoy/config/overload/v3/overload.upb_minitable.h" , "envoy/config/rbac/v3/rbac.upb.h" , "envoy/config/rbac/v3/rbac.upb_minitable.h" , "envoy/config/route/v3/route.upb.h" , "envoy/config/route/v3/route.upb_minitable.h" , "envoy/config/route/v3/route_components.upb.h" , "envoy/config/route/v3/route_components.upb_minitable.h" , "envoy/config/route/v3/scoped_route.upb.h" , "envoy/config/route/v3/scoped_route.upb_minitable.h" , "envoy/config/tap/v3/common.upb.h" , "envoy/config/tap/v3/common.upb_minitable.h" , "envoy/config/trace/v3/datadog.upb.h" , "envoy/config/trace/v3/datadog.upb_minitable.h" , "envoy/config/trace/v3/dynamic_ot.upb.h" , "envoy/config/trace/v3/dynamic_ot.upb_minitable.h" , "envoy/config/trace/v3/http_tracer.upb.h" , "envoy/config/trace/v3/http_tracer.upb_minitable.h" , "envoy/config/trace/v3/lightstep.upb.h" , "envoy/config/trace/v3/lightstep.upb_minitable.h" , "envoy/config/trace/v3/opentelemetry.upb.h" , "envoy/config/trace/v3/opentelemetry.upb_minitable.h" , "envoy/config/trace/v3/service.upb.h" , "envoy/config/trace/v3/service.upb_minitable.h" , "envoy/config/trace/v3/skywalking.upb.h" , "envoy/config/trace/v3/skywalking.upb_minitable.h" , "envoy/config/trace/v3/trace.upb.h" , "envoy/config/trace/v3/trace.upb_minitable.h" , "envoy/config/trace/v3/xray.upb.h" , "envoy/config/trace/v3/xray.upb_minitable.h" , "envoy/config/trace/v3/zipkin.upb.h" , "envoy/config/trace/v3/zipkin.upb_minitable.h" , "envoy/data/accesslog/v3/accesslog.upb.h" , "envoy/data/accesslog/v3/accesslog.upb_minitable.h" , "envoy/extensions/clusters/aggregate/v3/cluster.upb.h" , "envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.h" , "envoy/extensions/filters/common/fault/v3/fault.upb.h" , "envoy/extensions/filters/common/fault/v3/fault.upb_minitable.h" , "envoy/extensions/filters/http/fault/v3/fault.upb.h" , "envoy/extensions/filters/http/fault/v3/fault.upb_minitable.h" , "envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb.h" , "envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.h" , "envoy/extensions/filters/http/rbac/v3/rbac.upb.h" , "envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.h" , "envoy/extensions/filters/http/router/v3/router.upb.h" , "envoy/extensions/filters/http/router/v3/router.upb_minitable.h" , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h" , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.h" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.h" , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h" , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.h" , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h" , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.h" , "envoy/extensions/load_balancing_policies/common/v3/common.upb.h" , "envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.h" , "envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb.h" , "envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.h" , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h" , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.h" , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h" , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.h" , "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb.h" , "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.h" , "envoy/extensions/transport_sockets/tls/v3/cert.upb.h" , "envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.h" , "envoy/extensions/transport_sockets/tls/v3/common.upb.h" , "envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.h" , "envoy/extensions/transport_sockets/tls/v3/secret.upb.h" , "envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.h" , "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" , "envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.h" , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h" , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.h" , "envoy/extensions/upstreams/http/v3/http_protocol_options.upb.h" , "envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.h" , "envoy/service/discovery/v3/ads.upb.h" , "envoy/service/discovery/v3/ads.upb_minitable.h" , "envoy/service/discovery/v3/discovery.upb.h" , "envoy/service/discovery/v3/discovery.upb_minitable.h" , "envoy/service/load_stats/v3/lrs.upb.h" , "envoy/service/load_stats/v3/lrs.upb_minitable.h" , "envoy/service/status/v3/csds.upb.h" , "envoy/service/status/v3/csds.upb_minitable.h" , "envoy/type/http/v3/cookie.upb.h" , "envoy/type/http/v3/cookie.upb_minitable.h" , "envoy/type/http/v3/path_transformation.upb.h" , "envoy/type/http/v3/path_transformation.upb_minitable.h" , "envoy/type/matcher/v3/filter_state.upb.h" , "envoy/type/matcher/v3/filter_state.upb_minitable.h" , "envoy/type/matcher/v3/http_inputs.upb.h" , "envoy/type/matcher/v3/http_inputs.upb_minitable.h" , "envoy/type/matcher/v3/metadata.upb.h" , "envoy/type/matcher/v3/metadata.upb_minitable.h" , "envoy/type/matcher/v3/node.upb.h" , "envoy/type/matcher/v3/node.upb_minitable.h" , "envoy/type/matcher/v3/number.upb.h" , "envoy/type/matcher/v3/number.upb_minitable.h" , "envoy/type/matcher/v3/path.upb.h" , "envoy/type/matcher/v3/path.upb_minitable.h" , "envoy/type/matcher/v3/regex.upb.h" , "envoy/type/matcher/v3/regex.upb_minitable.h" , "envoy/type/matcher/v3/status_code_input.upb.h" , "envoy/type/matcher/v3/status_code_input.upb_minitable.h" , "envoy/type/matcher/v3/string.upb.h" , "envoy/type/matcher/v3/string.upb_minitable.h" , "envoy/type/matcher/v3/struct.upb.h" , "envoy/type/matcher/v3/struct.upb_minitable.h" , "envoy/type/matcher/v3/value.upb.h" , "envoy/type/matcher/v3/value.upb_minitable.h" , "envoy/type/metadata/v3/metadata.upb.h" , "envoy/type/metadata/v3/metadata.upb_minitable.h" , "envoy/type/tracing/v3/custom_tag.upb.h" , "envoy/type/tracing/v3/custom_tag.upb_minitable.h" , "envoy/type/v3/hash_policy.upb.h" , "envoy/type/v3/hash_policy.upb_minitable.h" , "envoy/type/v3/http.upb.h" , "envoy/type/v3/http.upb_minitable.h" , "envoy/type/v3/http_status.upb.h" , "envoy/type/v3/http_status.upb_minitable.h" , "envoy/type/v3/percent.upb.h" , "envoy/type/v3/percent.upb_minitable.h" , "envoy/type/v3/range.upb.h" , "envoy/type/v3/range.upb_minitable.h" , "envoy/type/v3/ratelimit_strategy.upb.h" , "envoy/type/v3/ratelimit_strategy.upb_minitable.h" , "envoy/type/v3/ratelimit_unit.upb.h" , "envoy/type/v3/ratelimit_unit.upb_minitable.h" , "envoy/type/v3/semantic_version.upb.h" , "envoy/type/v3/semantic_version.upb_minitable.h" , "envoy/type/v3/token_bucket.upb.h" , "envoy/type/v3/token_bucket.upb_minitable.h" , "google/api/annotations.upb.h" , "google/api/annotations.upb_minitable.h" , "google/api/expr/v1alpha1/checked.upb.h" , "google/api/expr/v1alpha1/checked.upb_minitable.h" , "google/api/expr/v1alpha1/syntax.upb.h" , "google/api/expr/v1alpha1/syntax.upb_minitable.h" , "google/api/http.upb.h" , "google/api/http.upb_minitable.h" , "google/api/httpbody.upb.h" , "google/api/httpbody.upb_minitable.h" , "google/protobuf/any.upb.h" , "google/protobuf/any.upb_minitable.h" , "google/protobuf/descriptor.upb.h" , "google/protobuf/descriptor.upb_minitable.h" , "google/protobuf/duration.upb.h" , "google/protobuf/duration.upb_minitable.h" , "google/protobuf/empty.upb.h" , "google/protobuf/empty.upb_minitable.h" , "google/protobuf/struct.upb.h" , "google/protobuf/struct.upb_minitable.h" , "google/protobuf/timestamp.upb.h" , "google/protobuf/timestamp.upb_minitable.h" , "google/protobuf/wrappers.upb.h" , "google/protobuf/wrappers.upb_minitable.h" , "google/rpc/status.upb.h" , "google/rpc/status.upb_minitable.h" , "src/proto/grpc/gcp/altscontext.upb.h" , "src/proto/grpc/gcp/altscontext.upb_minitable.h" , "src/proto/grpc/gcp/handshaker.upb.h" , "src/proto/grpc/gcp/handshaker.upb_minitable.h" , "src/proto/grpc/gcp/transport_security_common.upb.h" , "src/proto/grpc/gcp/transport_security_common.upb_minitable.h" , "src/proto/grpc/health/v1/health.upb.h" , "src/proto/grpc/health/v1/health.upb_minitable.h" , "src/proto/grpc/lb/v1/load_balancer.upb.h" , "src/proto/grpc/lb/v1/load_balancer.upb_minitable.h" , "src/proto/grpc/lookup/v1/rls.upb.h" , "src/proto/grpc/lookup/v1/rls.upb_minitable.h" , "src/proto/grpc/lookup/v1/rls_config.upb.h" , "src/proto/grpc/lookup/v1/rls_config.upb_minitable.h" , "udpa/annotations/migrate.upb.h" , "udpa/annotations/migrate.upb_minitable.h" , "udpa/annotations/security.upb.h" , "udpa/annotations/security.upb_minitable.h" , "udpa/annotations/sensitive.upb.h" , "udpa/annotations/sensitive.upb_minitable.h" , "udpa/annotations/status.upb.h" , "udpa/annotations/status.upb_minitable.h" , "udpa/annotations/versioning.upb.h" , "udpa/annotations/versioning.upb_minitable.h" , "validate/validate.upb.h" , "validate/validate.upb_minitable.h" , "xds/annotations/v3/migrate.upb.h" , "xds/annotations/v3/migrate.upb_minitable.h" , "xds/annotations/v3/security.upb.h" , "xds/annotations/v3/security.upb_minitable.h" , "xds/annotations/v3/sensitive.upb.h" , "xds/annotations/v3/sensitive.upb_minitable.h" , "xds/annotations/v3/status.upb.h" , "xds/annotations/v3/status.upb_minitable.h" , "xds/annotations/v3/versioning.upb.h" , "xds/annotations/v3/versioning.upb_minitable.h" , "xds/core/v3/authority.upb.h" , "xds/core/v3/authority.upb_minitable.h" , "xds/core/v3/cidr.upb.h" , "xds/core/v3/cidr.upb_minitable.h" , "xds/core/v3/collection_entry.upb.h" , "xds/core/v3/collection_entry.upb_minitable.h" , "xds/core/v3/context_params.upb.h" , "xds/core/v3/context_params.upb_minitable.h" , "xds/core/v3/extension.upb.h" , "xds/core/v3/extension.upb_minitable.h" , "xds/core/v3/resource.upb.h" , "xds/core/v3/resource.upb_minitable.h" , "xds/core/v3/resource_locator.upb.h" , "xds/core/v3/resource_locator.upb_minitable.h" , "xds/core/v3/resource_name.upb.h" , "xds/core/v3/resource_name.upb_minitable.h" , "xds/data/orca/v3/orca_load_report.upb.h" , "xds/data/orca/v3/orca_load_report.upb_minitable.h" , "xds/service/orca/v3/orca.upb.h" , "xds/service/orca/v3/orca.upb_minitable.h" , "xds/type/matcher/v3/cel.upb.h" , "xds/type/matcher/v3/cel.upb_minitable.h" , "xds/type/matcher/v3/domain.upb.h" , "xds/type/matcher/v3/domain.upb_minitable.h" , "xds/type/matcher/v3/http_inputs.upb.h" , "xds/type/matcher/v3/http_inputs.upb_minitable.h" , "xds/type/matcher/v3/ip.upb.h" , "xds/type/matcher/v3/ip.upb_minitable.h" , "xds/type/matcher/v3/matcher.upb.h" , "xds/type/matcher/v3/matcher.upb_minitable.h" , "xds/type/matcher/v3/range.upb.h" , "xds/type/matcher/v3/range.upb_minitable.h" , "xds/type/matcher/v3/regex.upb.h" , "xds/type/matcher/v3/regex.upb_minitable.h" , "xds/type/matcher/v3/string.upb.h" , "xds/type/matcher/v3/string.upb_minitable.h" , "xds/type/v3/cel.upb.h" , "xds/type/v3/cel.upb_minitable.h" , "xds/type/v3/range.upb.h" , "xds/type/v3/range.upb_minitable.h" , "xds/type/v3/typed_struct.upb.h" , "xds/type/v3/typed_struct.upb_minitable.h" ] , "deps": [["third_party/upb", "generated_code_support"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/ext/upbdefs-gen/000077500000000000000000000000001516554100600264225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/core/ext/upbdefs-gen/TARGETS.grpc000066400000000000000000000365261516554100600304240ustar00rootroot00000000000000{ "upbdefs-gen-lib": { "type": ["@", "rules", "CC", "library"] , "name": ["upbdefs-gen-lib"] , "pure C": ["YES"] , "hdrs": [ "envoy/admin/v3/certs.upbdefs.h" , "envoy/admin/v3/clusters.upbdefs.h" , "envoy/admin/v3/config_dump.upbdefs.h" , "envoy/admin/v3/config_dump_shared.upbdefs.h" , "envoy/admin/v3/init_dump.upbdefs.h" , "envoy/admin/v3/listeners.upbdefs.h" , "envoy/admin/v3/memory.upbdefs.h" , "envoy/admin/v3/metrics.upbdefs.h" , "envoy/admin/v3/mutex_stats.upbdefs.h" , "envoy/admin/v3/server_info.upbdefs.h" , "envoy/admin/v3/tap.upbdefs.h" , "envoy/annotations/deprecation.upbdefs.h" , "envoy/annotations/resource.upbdefs.h" , "envoy/config/accesslog/v3/accesslog.upbdefs.h" , "envoy/config/bootstrap/v3/bootstrap.upbdefs.h" , "envoy/config/cluster/v3/circuit_breaker.upbdefs.h" , "envoy/config/cluster/v3/cluster.upbdefs.h" , "envoy/config/cluster/v3/filter.upbdefs.h" , "envoy/config/cluster/v3/outlier_detection.upbdefs.h" , "envoy/config/common/matcher/v3/matcher.upbdefs.h" , "envoy/config/core/v3/address.upbdefs.h" , "envoy/config/core/v3/backoff.upbdefs.h" , "envoy/config/core/v3/base.upbdefs.h" , "envoy/config/core/v3/config_source.upbdefs.h" , "envoy/config/core/v3/event_service_config.upbdefs.h" , "envoy/config/core/v3/extension.upbdefs.h" , "envoy/config/core/v3/grpc_method_list.upbdefs.h" , "envoy/config/core/v3/grpc_service.upbdefs.h" , "envoy/config/core/v3/health_check.upbdefs.h" , "envoy/config/core/v3/http_service.upbdefs.h" , "envoy/config/core/v3/http_uri.upbdefs.h" , "envoy/config/core/v3/protocol.upbdefs.h" , "envoy/config/core/v3/proxy_protocol.upbdefs.h" , "envoy/config/core/v3/resolver.upbdefs.h" , "envoy/config/core/v3/socket_cmsg_headers.upbdefs.h" , "envoy/config/core/v3/socket_option.upbdefs.h" , "envoy/config/core/v3/substitution_format_string.upbdefs.h" , "envoy/config/core/v3/udp_socket_config.upbdefs.h" , "envoy/config/endpoint/v3/endpoint.upbdefs.h" , "envoy/config/endpoint/v3/endpoint_components.upbdefs.h" , "envoy/config/endpoint/v3/load_report.upbdefs.h" , "envoy/config/listener/v3/api_listener.upbdefs.h" , "envoy/config/listener/v3/listener.upbdefs.h" , "envoy/config/listener/v3/listener_components.upbdefs.h" , "envoy/config/listener/v3/quic_config.upbdefs.h" , "envoy/config/listener/v3/udp_listener_config.upbdefs.h" , "envoy/config/metrics/v3/metrics_service.upbdefs.h" , "envoy/config/metrics/v3/stats.upbdefs.h" , "envoy/config/overload/v3/overload.upbdefs.h" , "envoy/config/rbac/v3/rbac.upbdefs.h" , "envoy/config/route/v3/route.upbdefs.h" , "envoy/config/route/v3/route_components.upbdefs.h" , "envoy/config/route/v3/scoped_route.upbdefs.h" , "envoy/config/tap/v3/common.upbdefs.h" , "envoy/config/trace/v3/datadog.upbdefs.h" , "envoy/config/trace/v3/dynamic_ot.upbdefs.h" , "envoy/config/trace/v3/http_tracer.upbdefs.h" , "envoy/config/trace/v3/lightstep.upbdefs.h" , "envoy/config/trace/v3/opentelemetry.upbdefs.h" , "envoy/config/trace/v3/service.upbdefs.h" , "envoy/config/trace/v3/skywalking.upbdefs.h" , "envoy/config/trace/v3/trace.upbdefs.h" , "envoy/config/trace/v3/xray.upbdefs.h" , "envoy/config/trace/v3/zipkin.upbdefs.h" , "envoy/data/accesslog/v3/accesslog.upbdefs.h" , "envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.h" , "envoy/extensions/filters/common/fault/v3/fault.upbdefs.h" , "envoy/extensions/filters/http/fault/v3/fault.upbdefs.h" , "envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.h" , "envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.h" , "envoy/extensions/filters/http/router/v3/router.upbdefs.h" , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h" , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h" , "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.h" , "envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.h" , "envoy/extensions/transport_sockets/tls/v3/common.upbdefs.h" , "envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.h" , "envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h" , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.h" , "envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.h" , "envoy/service/discovery/v3/ads.upbdefs.h" , "envoy/service/discovery/v3/discovery.upbdefs.h" , "envoy/service/load_stats/v3/lrs.upbdefs.h" , "envoy/service/status/v3/csds.upbdefs.h" , "envoy/type/http/v3/cookie.upbdefs.h" , "envoy/type/http/v3/path_transformation.upbdefs.h" , "envoy/type/matcher/v3/filter_state.upbdefs.h" , "envoy/type/matcher/v3/http_inputs.upbdefs.h" , "envoy/type/matcher/v3/metadata.upbdefs.h" , "envoy/type/matcher/v3/node.upbdefs.h" , "envoy/type/matcher/v3/number.upbdefs.h" , "envoy/type/matcher/v3/path.upbdefs.h" , "envoy/type/matcher/v3/regex.upbdefs.h" , "envoy/type/matcher/v3/status_code_input.upbdefs.h" , "envoy/type/matcher/v3/string.upbdefs.h" , "envoy/type/matcher/v3/struct.upbdefs.h" , "envoy/type/matcher/v3/value.upbdefs.h" , "envoy/type/metadata/v3/metadata.upbdefs.h" , "envoy/type/tracing/v3/custom_tag.upbdefs.h" , "envoy/type/v3/hash_policy.upbdefs.h" , "envoy/type/v3/http.upbdefs.h" , "envoy/type/v3/http_status.upbdefs.h" , "envoy/type/v3/percent.upbdefs.h" , "envoy/type/v3/range.upbdefs.h" , "envoy/type/v3/ratelimit_strategy.upbdefs.h" , "envoy/type/v3/ratelimit_unit.upbdefs.h" , "envoy/type/v3/semantic_version.upbdefs.h" , "envoy/type/v3/token_bucket.upbdefs.h" , "google/api/annotations.upbdefs.h" , "google/api/expr/v1alpha1/checked.upbdefs.h" , "google/api/expr/v1alpha1/syntax.upbdefs.h" , "google/api/http.upbdefs.h" , "google/api/httpbody.upbdefs.h" , "google/protobuf/any.upbdefs.h" , "google/protobuf/descriptor.upbdefs.h" , "google/protobuf/duration.upbdefs.h" , "google/protobuf/empty.upbdefs.h" , "google/protobuf/struct.upbdefs.h" , "google/protobuf/timestamp.upbdefs.h" , "google/protobuf/wrappers.upbdefs.h" , "google/rpc/status.upbdefs.h" , "src/proto/grpc/lookup/v1/rls_config.upbdefs.h" , "udpa/annotations/migrate.upbdefs.h" , "udpa/annotations/security.upbdefs.h" , "udpa/annotations/sensitive.upbdefs.h" , "udpa/annotations/status.upbdefs.h" , "udpa/annotations/versioning.upbdefs.h" , "validate/validate.upbdefs.h" , "xds/annotations/v3/migrate.upbdefs.h" , "xds/annotations/v3/security.upbdefs.h" , "xds/annotations/v3/sensitive.upbdefs.h" , "xds/annotations/v3/status.upbdefs.h" , "xds/annotations/v3/versioning.upbdefs.h" , "xds/core/v3/authority.upbdefs.h" , "xds/core/v3/cidr.upbdefs.h" , "xds/core/v3/collection_entry.upbdefs.h" , "xds/core/v3/context_params.upbdefs.h" , "xds/core/v3/extension.upbdefs.h" , "xds/core/v3/resource.upbdefs.h" , "xds/core/v3/resource_locator.upbdefs.h" , "xds/core/v3/resource_name.upbdefs.h" , "xds/type/matcher/v3/cel.upbdefs.h" , "xds/type/matcher/v3/domain.upbdefs.h" , "xds/type/matcher/v3/http_inputs.upbdefs.h" , "xds/type/matcher/v3/ip.upbdefs.h" , "xds/type/matcher/v3/matcher.upbdefs.h" , "xds/type/matcher/v3/range.upbdefs.h" , "xds/type/matcher/v3/regex.upbdefs.h" , "xds/type/matcher/v3/string.upbdefs.h" , "xds/type/v3/cel.upbdefs.h" , "xds/type/v3/range.upbdefs.h" , "xds/type/v3/typed_struct.upbdefs.h" ] , "srcs": [ "envoy/admin/v3/certs.upbdefs.c" , "envoy/admin/v3/clusters.upbdefs.c" , "envoy/admin/v3/config_dump.upbdefs.c" , "envoy/admin/v3/config_dump_shared.upbdefs.c" , "envoy/admin/v3/init_dump.upbdefs.c" , "envoy/admin/v3/listeners.upbdefs.c" , "envoy/admin/v3/memory.upbdefs.c" , "envoy/admin/v3/metrics.upbdefs.c" , "envoy/admin/v3/mutex_stats.upbdefs.c" , "envoy/admin/v3/server_info.upbdefs.c" , "envoy/admin/v3/tap.upbdefs.c" , "envoy/annotations/deprecation.upbdefs.c" , "envoy/annotations/resource.upbdefs.c" , "envoy/config/accesslog/v3/accesslog.upbdefs.c" , "envoy/config/bootstrap/v3/bootstrap.upbdefs.c" , "envoy/config/cluster/v3/circuit_breaker.upbdefs.c" , "envoy/config/cluster/v3/cluster.upbdefs.c" , "envoy/config/cluster/v3/filter.upbdefs.c" , "envoy/config/cluster/v3/outlier_detection.upbdefs.c" , "envoy/config/common/matcher/v3/matcher.upbdefs.c" , "envoy/config/core/v3/address.upbdefs.c" , "envoy/config/core/v3/backoff.upbdefs.c" , "envoy/config/core/v3/base.upbdefs.c" , "envoy/config/core/v3/config_source.upbdefs.c" , "envoy/config/core/v3/event_service_config.upbdefs.c" , "envoy/config/core/v3/extension.upbdefs.c" , "envoy/config/core/v3/grpc_method_list.upbdefs.c" , "envoy/config/core/v3/grpc_service.upbdefs.c" , "envoy/config/core/v3/health_check.upbdefs.c" , "envoy/config/core/v3/http_service.upbdefs.c" , "envoy/config/core/v3/http_uri.upbdefs.c" , "envoy/config/core/v3/protocol.upbdefs.c" , "envoy/config/core/v3/proxy_protocol.upbdefs.c" , "envoy/config/core/v3/resolver.upbdefs.c" , "envoy/config/core/v3/socket_cmsg_headers.upbdefs.c" , "envoy/config/core/v3/socket_option.upbdefs.c" , "envoy/config/core/v3/substitution_format_string.upbdefs.c" , "envoy/config/core/v3/udp_socket_config.upbdefs.c" , "envoy/config/endpoint/v3/endpoint.upbdefs.c" , "envoy/config/endpoint/v3/endpoint_components.upbdefs.c" , "envoy/config/endpoint/v3/load_report.upbdefs.c" , "envoy/config/listener/v3/api_listener.upbdefs.c" , "envoy/config/listener/v3/listener.upbdefs.c" , "envoy/config/listener/v3/listener_components.upbdefs.c" , "envoy/config/listener/v3/quic_config.upbdefs.c" , "envoy/config/listener/v3/udp_listener_config.upbdefs.c" , "envoy/config/metrics/v3/metrics_service.upbdefs.c" , "envoy/config/metrics/v3/stats.upbdefs.c" , "envoy/config/overload/v3/overload.upbdefs.c" , "envoy/config/rbac/v3/rbac.upbdefs.c" , "envoy/config/route/v3/route.upbdefs.c" , "envoy/config/route/v3/route_components.upbdefs.c" , "envoy/config/route/v3/scoped_route.upbdefs.c" , "envoy/config/tap/v3/common.upbdefs.c" , "envoy/config/trace/v3/datadog.upbdefs.c" , "envoy/config/trace/v3/dynamic_ot.upbdefs.c" , "envoy/config/trace/v3/http_tracer.upbdefs.c" , "envoy/config/trace/v3/lightstep.upbdefs.c" , "envoy/config/trace/v3/opentelemetry.upbdefs.c" , "envoy/config/trace/v3/service.upbdefs.c" , "envoy/config/trace/v3/skywalking.upbdefs.c" , "envoy/config/trace/v3/trace.upbdefs.c" , "envoy/config/trace/v3/xray.upbdefs.c" , "envoy/config/trace/v3/zipkin.upbdefs.c" , "envoy/data/accesslog/v3/accesslog.upbdefs.c" , "envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c" , "envoy/extensions/filters/common/fault/v3/fault.upbdefs.c" , "envoy/extensions/filters/http/fault/v3/fault.upbdefs.c" , "envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.c" , "envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c" , "envoy/extensions/filters/http/router/v3/router.upbdefs.c" , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c" , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c" , "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.c" , "envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c" , "envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c" , "envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c" , "envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c" , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c" , "envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.c" , "envoy/service/discovery/v3/ads.upbdefs.c" , "envoy/service/discovery/v3/discovery.upbdefs.c" , "envoy/service/load_stats/v3/lrs.upbdefs.c" , "envoy/service/status/v3/csds.upbdefs.c" , "envoy/type/http/v3/cookie.upbdefs.c" , "envoy/type/http/v3/path_transformation.upbdefs.c" , "envoy/type/matcher/v3/filter_state.upbdefs.c" , "envoy/type/matcher/v3/http_inputs.upbdefs.c" , "envoy/type/matcher/v3/metadata.upbdefs.c" , "envoy/type/matcher/v3/node.upbdefs.c" , "envoy/type/matcher/v3/number.upbdefs.c" , "envoy/type/matcher/v3/path.upbdefs.c" , "envoy/type/matcher/v3/regex.upbdefs.c" , "envoy/type/matcher/v3/status_code_input.upbdefs.c" , "envoy/type/matcher/v3/string.upbdefs.c" , "envoy/type/matcher/v3/struct.upbdefs.c" , "envoy/type/matcher/v3/value.upbdefs.c" , "envoy/type/metadata/v3/metadata.upbdefs.c" , "envoy/type/tracing/v3/custom_tag.upbdefs.c" , "envoy/type/v3/hash_policy.upbdefs.c" , "envoy/type/v3/http.upbdefs.c" , "envoy/type/v3/http_status.upbdefs.c" , "envoy/type/v3/percent.upbdefs.c" , "envoy/type/v3/range.upbdefs.c" , "envoy/type/v3/ratelimit_strategy.upbdefs.c" , "envoy/type/v3/ratelimit_unit.upbdefs.c" , "envoy/type/v3/semantic_version.upbdefs.c" , "envoy/type/v3/token_bucket.upbdefs.c" , "google/api/annotations.upbdefs.c" , "google/api/expr/v1alpha1/checked.upbdefs.c" , "google/api/expr/v1alpha1/syntax.upbdefs.c" , "google/api/http.upbdefs.c" , "google/api/httpbody.upbdefs.c" , "google/protobuf/any.upbdefs.c" , "google/protobuf/descriptor.upbdefs.c" , "google/protobuf/duration.upbdefs.c" , "google/protobuf/empty.upbdefs.c" , "google/protobuf/struct.upbdefs.c" , "google/protobuf/timestamp.upbdefs.c" , "google/protobuf/wrappers.upbdefs.c" , "google/rpc/status.upbdefs.c" , "src/proto/grpc/lookup/v1/rls_config.upbdefs.c" , "udpa/annotations/migrate.upbdefs.c" , "udpa/annotations/security.upbdefs.c" , "udpa/annotations/sensitive.upbdefs.c" , "udpa/annotations/status.upbdefs.c" , "udpa/annotations/versioning.upbdefs.c" , "validate/validate.upbdefs.c" , "xds/annotations/v3/migrate.upbdefs.c" , "xds/annotations/v3/security.upbdefs.c" , "xds/annotations/v3/sensitive.upbdefs.c" , "xds/annotations/v3/status.upbdefs.c" , "xds/annotations/v3/versioning.upbdefs.c" , "xds/core/v3/authority.upbdefs.c" , "xds/core/v3/cidr.upbdefs.c" , "xds/core/v3/collection_entry.upbdefs.c" , "xds/core/v3/context_params.upbdefs.c" , "xds/core/v3/extension.upbdefs.c" , "xds/core/v3/resource.upbdefs.c" , "xds/core/v3/resource_locator.upbdefs.c" , "xds/core/v3/resource_name.upbdefs.c" , "xds/type/matcher/v3/cel.upbdefs.c" , "xds/type/matcher/v3/domain.upbdefs.c" , "xds/type/matcher/v3/http_inputs.upbdefs.c" , "xds/type/matcher/v3/ip.upbdefs.c" , "xds/type/matcher/v3/matcher.upbdefs.c" , "xds/type/matcher/v3/range.upbdefs.c" , "xds/type/matcher/v3/regex.upbdefs.c" , "xds/type/matcher/v3/string.upbdefs.c" , "xds/type/v3/cel.upbdefs.c" , "xds/type/v3/range.upbdefs.c" , "xds/type/v3/typed_struct.upbdefs.c" ] , "deps": [ ["src/core/ext/upb-gen", "upb-gen-lib"] , ["third_party/upb", "reflection"] , ["third_party/upb", "reflection_internal"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/google/000077500000000000000000000000001516554100600237475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/google/protobuf/000077500000000000000000000000001516554100600256075ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/google/protobuf/TARGETS.protobuf000066400000000000000000000537341516554100600305160ustar00rootroot00000000000000{ "well_known_protos": { "type": "install" , "deps": ["compiler/plugin.proto", "descriptor.proto", "well_known_type_protos"] } , "well_known_type_protos": { "type": "install" , "deps": [ "any.proto" , "api.proto" , "duration.proto" , "empty.proto" , "field_mask.proto" , "source_context.proto" , "struct.proto" , "timestamp.proto" , "type.proto" , "wrappers.proto" ] } , "protobuf_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "well_known_protos" , "any.pb.h" , "api.pb.h" , "duration.pb.h" , "empty.pb.h" , "field_mask.pb.h" , "source_context.pb.h" , "struct.pb.h" , "timestamp.pb.h" , "type.pb.h" , "wrappers.pb.h" , "any.h" , "arena.h" , "arena_align.h" , "arena_allocation_policy.h" , "arena_cleanup.h" , "arenastring.h" , "arenaz_sampler.h" , "compiler/importer.h" , "compiler/parser.h" , "cpp_edition_defaults.h" , "cpp_features.pb.h" , "descriptor.h" , "descriptor.pb.h" , "descriptor_database.h" , "descriptor_legacy.h" , "descriptor_lite.h" , "descriptor_visitor.h" , "dynamic_message.h" , "endian.h" , "explicitly_constructed.h" , "extension_set.h" , "extension_set_inl.h" , "feature_resolver.h" , "field_access_listener.h" , "generated_enum_reflection.h" , "generated_enum_util.h" , "generated_message_bases.h" , "generated_message_reflection.h" , "generated_message_tctable_decl.h" , "generated_message_tctable_gen.h" , "generated_message_tctable_impl.h" , "generated_message_util.h" , "has_bits.h" , "implicit_weak_message.h" , "inlined_string_field.h" , "internal_visibility.h" , "io/coded_stream.h" , "io/gzip_stream.h" , "io/io_win32.h" , "io/printer.h" , "io/strtod.h" , "io/tokenizer.h" , "io/zero_copy_sink.h" , "io/zero_copy_stream.h" , "io/zero_copy_stream_impl.h" , "io/zero_copy_stream_impl_lite.h" , "json/internal/descriptor_traits.h" , "json/internal/lexer.h" , "json/internal/message_path.h" , "json/internal/parser.h" , "json/internal/parser_traits.h" , "json/internal/unparser.h" , "json/internal/unparser_traits.h" , "json/internal/untyped_message.h" , "json/internal/writer.h" , "json/internal/zero_copy_buffered_stream.h" , "json/json.h" , "map.h" , "map_entry.h" , "map_field.h" , "map_field_inl.h" , "map_field_lite.h" , "map_type_handler.h" , "message.h" , "message_lite.h" , "metadata.h" , "metadata_lite.h" , "parse_context.h" , "port.h" , "port_def.inc" , "port_undef.inc" , "raw_ptr.h" , "reflection.h" , "reflection_internal.h" , "reflection_mode.h" , "reflection_ops.h" , "reflection_visit_field_info.h" , "reflection_visit_fields.h" , "repeated_field.h" , "repeated_ptr_field.h" , "runtime_version.h" , "serial_arena.h" , "service.h" , "string_block.h" , "stubs/callback.h" , "stubs/common.h" , "stubs/platform_macros.h" , "stubs/port.h" , "stubs/status_macros.h" , "text_format.h" , "thread_safe_arena.h" , "unknown_field_set.h" , "util/delimited_message_util.h" , "util/field_comparator.h" , "util/field_mask_util.h" , "util/json_util.h" , "util/message_differencer.h" , "util/time_util.h" , "util/type_resolver.h" , "util/type_resolver_util.h" , "varint_shuffle.h" , "wire_format.h" , "wire_format_lite.h" ] , "stage": ["google", "protobuf"] } , "protobuf_lite_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "any.h" , "arena.h" , "arena_align.h" , "arena_allocation_policy.h" , "arena_cleanup.h" , "arenastring.h" , "arenaz_sampler.h" , "descriptor_lite.h" , "endian.h" , "explicitly_constructed.h" , "extension_set.h" , "extension_set_inl.h" , "generated_enum_util.h" , "generated_message_tctable_decl.h" , "generated_message_tctable_impl.h" , "generated_message_util.h" , "has_bits.h" , "implicit_weak_message.h" , "inlined_string_field.h" , "internal_visibility.h" , "io/coded_stream.h" , "io/io_win32.h" , "io/zero_copy_stream.h" , "io/zero_copy_stream_impl.h" , "io/zero_copy_stream_impl_lite.h" , "map.h" , "map_field_lite.h" , "map_type_handler.h" , "message_lite.h" , "metadata_lite.h" , "parse_context.h" , "port.h" , "port_def.inc" , "port_undef.inc" , "raw_ptr.h" , "repeated_field.h" , "repeated_ptr_field.h" , "runtime_version.h" , "serial_arena.h" , "string_block.h" , "stubs/callback.h" , "stubs/common.h" , "stubs/platform_macros.h" , "stubs/port.h" , "stubs/status_macros.h" , "thread_safe_arena.h" , "varint_shuffle.h" , "wire_format_lite.h" ] , "stage": ["google", "protobuf"] } , "protoc_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "compiler/code_generator.h" , "compiler/code_generator_lite.h" , "compiler/command_line_interface.h" , "compiler/cpp/enum.h" , "compiler/cpp/extension.h" , "compiler/cpp/field.h" , "compiler/cpp/field_generators/generators.h" , "compiler/cpp/file.h" , "compiler/cpp/generator.h" , "compiler/cpp/helpers.h" , "compiler/cpp/ifndef_guard.h" , "compiler/cpp/message.h" , "compiler/cpp/message_layout_helper.h" , "compiler/cpp/names.h" , "compiler/cpp/namespace_printer.h" , "compiler/cpp/options.h" , "compiler/cpp/padding_optimizer.h" , "compiler/cpp/parse_function_generator.h" , "compiler/cpp/service.h" , "compiler/cpp/tracker.h" , "compiler/csharp/csharp_doc_comment.h" , "compiler/csharp/csharp_enum.h" , "compiler/csharp/csharp_enum_field.h" , "compiler/csharp/csharp_field_base.h" , "compiler/csharp/csharp_generator.h" , "compiler/csharp/csharp_helpers.h" , "compiler/csharp/csharp_map_field.h" , "compiler/csharp/csharp_message.h" , "compiler/csharp/csharp_message_field.h" , "compiler/csharp/csharp_options.h" , "compiler/csharp/csharp_primitive_field.h" , "compiler/csharp/csharp_reflection_class.h" , "compiler/csharp/csharp_repeated_enum_field.h" , "compiler/csharp/csharp_repeated_message_field.h" , "compiler/csharp/csharp_repeated_primitive_field.h" , "compiler/csharp/csharp_source_generator_base.h" , "compiler/csharp/csharp_wrapper_field.h" , "compiler/csharp/names.h" , "compiler/java/context.h" , "compiler/java/doc_comment.h" , "compiler/java/field_common.h" , "compiler/java/file.h" , "compiler/java/full/enum.h" , "compiler/java/full/enum_field.h" , "compiler/java/full/extension.h" , "compiler/java/full/field_generator.h" , "compiler/java/full/generator_factory.h" , "compiler/java/full/make_field_gens.h" , "compiler/java/full/map_field.h" , "compiler/java/full/message.h" , "compiler/java/full/message_builder.h" , "compiler/java/full/message_field.h" , "compiler/java/full/primitive_field.h" , "compiler/java/full/service.h" , "compiler/java/full/string_field.h" , "compiler/java/generator.h" , "compiler/java/generator_common.h" , "compiler/java/generator_factory.h" , "compiler/java/helpers.h" , "compiler/java/internal_helpers.h" , "compiler/java/java_features.pb.h" , "compiler/java/lite/enum.h" , "compiler/java/lite/enum_field.h" , "compiler/java/lite/extension.h" , "compiler/java/lite/field_generator.h" , "compiler/java/lite/generator_factory.h" , "compiler/java/lite/make_field_gens.h" , "compiler/java/lite/map_field.h" , "compiler/java/lite/message.h" , "compiler/java/lite/message_builder.h" , "compiler/java/lite/message_field.h" , "compiler/java/lite/primitive_field.h" , "compiler/java/lite/string_field.h" , "compiler/java/message_serialization.h" , "compiler/java/name_resolver.h" , "compiler/java/names.h" , "compiler/java/options.h" , "compiler/java/shared_code_generator.h" , "compiler/kotlin/file.h" , "compiler/kotlin/generator.h" , "compiler/kotlin/message.h" , "compiler/objectivec/enum.h" , "compiler/objectivec/enum_field.h" , "compiler/objectivec/extension.h" , "compiler/objectivec/field.h" , "compiler/objectivec/file.h" , "compiler/objectivec/generator.h" , "compiler/objectivec/helpers.h" , "compiler/objectivec/import_writer.h" , "compiler/objectivec/line_consumer.h" , "compiler/objectivec/map_field.h" , "compiler/objectivec/message.h" , "compiler/objectivec/message_field.h" , "compiler/objectivec/names.h" , "compiler/objectivec/nsobject_methods.h" , "compiler/objectivec/oneof.h" , "compiler/objectivec/options.h" , "compiler/objectivec/primitive_field.h" , "compiler/objectivec/tf_decode_data.h" , "compiler/php/names.h" , "compiler/php/php_generator.h" , "compiler/plugin.h" , "compiler/plugin.pb.h" , "compiler/python/generator.h" , "compiler/python/helpers.h" , "compiler/python/pyi_generator.h" , "compiler/retention.h" , "compiler/ruby/ruby_generator.h" , "compiler/rust/accessors/accessor_case.h" , "compiler/rust/accessors/accessors.h" , "compiler/rust/accessors/default_value.h" , "compiler/rust/accessors/generator.h" , "compiler/rust/accessors/with_presence.h" , "compiler/rust/context.h" , "compiler/rust/crate_mapping.h" , "compiler/rust/enum.h" , "compiler/rust/generator.h" , "compiler/rust/message.h" , "compiler/rust/naming.h" , "compiler/rust/oneof.h" , "compiler/rust/relative_path.h" , "compiler/rust/rust_field_type.h" , "compiler/rust/rust_keywords.h" , "compiler/rust/upb_helpers.h" , "compiler/scc.h" , "compiler/subprocess.h" , "compiler/versions.h" , "compiler/zip_writer.h" , "testing/file.h" ] , "stage": ["google", "protobuf"] } , "libprotobuf_lite": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["TOOLCHAIN_CONFIG"] , "name": ["protobuf-lite"] , "cflags": { "type": "case" , "expr": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" , "default": "unknown" } , "case": { "gnu": [ "-Wno-sign-compare" , "-Wno-sign-conversion" , "-Wno-unused-function" , "-Wno-deprecated-declarations" ] } , "default": [ "-Wno-sign-compare" , "-Wno-sign-conversion" , "-Wno-unused-function" , "-Wno-zero-length-array" , "-Wno-deprecated-declarations" , "-Wno-invalid-noreturn" ] } , "hdrs": ["protobuf_lite_headers"] , "srcs": [ "any_lite.cc" , "arena.cc" , "arena_align.cc" , "arenastring.cc" , "arenaz_sampler.cc" , "extension_set.cc" , "generated_enum_util.cc" , "generated_message_tctable_lite.cc" , "generated_message_util.cc" , "implicit_weak_message.cc" , "inlined_string_field.cc" , "io/coded_stream.cc" , "io/io_win32.cc" , "io/zero_copy_stream.cc" , "io/zero_copy_stream_impl.cc" , "io/zero_copy_stream_impl_lite.cc" , "map.cc" , "message_lite.cc" , "parse_context.cc" , "port.cc" , "raw_ptr.cc" , "repeated_field.cc" , "repeated_ptr_field.cc" , "stubs/common.cc" , "wire_format_lite.cc" ] , "deps": [ ["@", "absl", "absl/base", "base"] , ["@", "absl", "absl/base", "config"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/base", "dynamic_annotations"] , ["@", "absl", "absl/base", "prefetch"] , ["@", "absl", "absl/container", "btree"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "log"] , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/numeric", "bits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "internal"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/utility", "if_constexpr"] , ["third_party/utf8_range", "utf8_validity"] ] } , "libprotobuf": { "type": ["@", "rules", "CC", "library"] , "name": ["protobuf"] , "hdrs": ["protobuf_headers"] , "srcs": [ "any.pb.cc" , "api.pb.cc" , "duration.pb.cc" , "empty.pb.cc" , "field_mask.pb.cc" , "source_context.pb.cc" , "struct.pb.cc" , "timestamp.pb.cc" , "type.pb.cc" , "wrappers.pb.cc" , "any.cc" , "any_lite.cc" , "arena.cc" , "arena_align.cc" , "arenastring.cc" , "arenaz_sampler.cc" , "compiler/importer.cc" , "compiler/parser.cc" , "cpp_features.pb.cc" , "descriptor.cc" , "descriptor.pb.cc" , "descriptor_database.cc" , "dynamic_message.cc" , "extension_set.cc" , "extension_set_heavy.cc" , "feature_resolver.cc" , "generated_enum_util.cc" , "generated_message_bases.cc" , "generated_message_reflection.cc" , "generated_message_tctable_full.cc" , "generated_message_tctable_gen.cc" , "generated_message_tctable_lite.cc" , "generated_message_util.cc" , "implicit_weak_message.cc" , "inlined_string_field.cc" , "io/coded_stream.cc" , "io/gzip_stream.cc" , "io/io_win32.cc" , "io/printer.cc" , "io/strtod.cc" , "io/tokenizer.cc" , "io/zero_copy_sink.cc" , "io/zero_copy_stream.cc" , "io/zero_copy_stream_impl.cc" , "io/zero_copy_stream_impl_lite.cc" , "json/internal/lexer.cc" , "json/internal/message_path.cc" , "json/internal/parser.cc" , "json/internal/unparser.cc" , "json/internal/untyped_message.cc" , "json/internal/writer.cc" , "json/internal/zero_copy_buffered_stream.cc" , "json/json.cc" , "map.cc" , "map_field.cc" , "message.cc" , "message_lite.cc" , "parse_context.cc" , "port.cc" , "raw_ptr.cc" , "reflection_mode.cc" , "reflection_ops.cc" , "repeated_field.cc" , "repeated_ptr_field.cc" , "service.cc" , "stubs/common.cc" , "text_format.cc" , "unknown_field_set.cc" , "util/delimited_message_util.cc" , "util/field_comparator.cc" , "util/field_mask_util.cc" , "util/message_differencer.cc" , "util/time_util.cc" , "util/type_resolver_util.cc" , "wire_format.cc" , "wire_format_lite.cc" ] , "deps": [ ["@", "absl", "absl/algorithm", "container"] , ["@", "absl", "absl/base", "base"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/base", "dynamic_annotations"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "btree"] , ["@", "absl", "absl/container", "fixed_array"] , ["@", "absl", "absl/container", "flat_hash_map"] , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/log", "absl_check"] , ["@", "absl", "absl/log", "absl_log"] , ["@", "absl", "absl/log", "die_if_null"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/numeric", "bits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "internal"] , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] , ["@", "absl", "absl/utility", "if_constexpr"] , ["@", "zlib", "", "zlib"] , ["", "libprotobuf_lite"] , ["third_party/utf8_range", "utf8_validity"] ] } , "libprotoc": { "type": ["@", "rules", "CC", "library"] , "name": ["libprotoc"] , "hdrs": ["protoc_headers", ["upb_generator", "names_hdrs"], ["upb", "port_hdrs"]] , "srcs": [ "compiler/main.cc" , "compiler/code_generator.cc" , "compiler/code_generator_lite.cc" , "compiler/command_line_interface.cc" , "compiler/cpp/enum.cc" , "compiler/cpp/extension.cc" , "compiler/cpp/field.cc" , "compiler/cpp/field_generators/cord_field.cc" , "compiler/cpp/field_generators/enum_field.cc" , "compiler/cpp/field_generators/map_field.cc" , "compiler/cpp/field_generators/message_field.cc" , "compiler/cpp/field_generators/primitive_field.cc" , "compiler/cpp/field_generators/string_field.cc" , "compiler/cpp/field_generators/string_view_field.cc" , "compiler/cpp/file.cc" , "compiler/cpp/generator.cc" , "compiler/cpp/helpers.cc" , "compiler/cpp/ifndef_guard.cc" , "compiler/cpp/message.cc" , "compiler/cpp/namespace_printer.cc" , "compiler/cpp/padding_optimizer.cc" , "compiler/cpp/parse_function_generator.cc" , "compiler/cpp/service.cc" , "compiler/cpp/tracker.cc" , "compiler/csharp/csharp_doc_comment.cc" , "compiler/csharp/csharp_enum.cc" , "compiler/csharp/csharp_enum_field.cc" , "compiler/csharp/csharp_field_base.cc" , "compiler/csharp/csharp_generator.cc" , "compiler/csharp/csharp_helpers.cc" , "compiler/csharp/csharp_map_field.cc" , "compiler/csharp/csharp_message.cc" , "compiler/csharp/csharp_message_field.cc" , "compiler/csharp/csharp_primitive_field.cc" , "compiler/csharp/csharp_reflection_class.cc" , "compiler/csharp/csharp_repeated_enum_field.cc" , "compiler/csharp/csharp_repeated_message_field.cc" , "compiler/csharp/csharp_repeated_primitive_field.cc" , "compiler/csharp/csharp_source_generator_base.cc" , "compiler/csharp/csharp_wrapper_field.cc" , "compiler/csharp/names.cc" , "compiler/java/context.cc" , "compiler/java/doc_comment.cc" , "compiler/java/field_common.cc" , "compiler/java/file.cc" , "compiler/java/full/enum.cc" , "compiler/java/full/enum_field.cc" , "compiler/java/full/extension.cc" , "compiler/java/full/generator_factory.cc" , "compiler/java/full/make_field_gens.cc" , "compiler/java/full/map_field.cc" , "compiler/java/full/message.cc" , "compiler/java/full/message_builder.cc" , "compiler/java/full/message_field.cc" , "compiler/java/full/primitive_field.cc" , "compiler/java/full/service.cc" , "compiler/java/full/string_field.cc" , "compiler/java/generator.cc" , "compiler/java/helpers.cc" , "compiler/java/internal_helpers.cc" , "compiler/java/java_features.pb.cc" , "compiler/java/lite/enum.cc" , "compiler/java/lite/enum_field.cc" , "compiler/java/lite/extension.cc" , "compiler/java/lite/generator_factory.cc" , "compiler/java/lite/make_field_gens.cc" , "compiler/java/lite/map_field.cc" , "compiler/java/lite/message.cc" , "compiler/java/lite/message_builder.cc" , "compiler/java/lite/message_field.cc" , "compiler/java/lite/primitive_field.cc" , "compiler/java/lite/string_field.cc" , "compiler/java/message_serialization.cc" , "compiler/java/name_resolver.cc" , "compiler/java/names.cc" , "compiler/java/shared_code_generator.cc" , "compiler/kotlin/file.cc" , "compiler/kotlin/generator.cc" , "compiler/kotlin/message.cc" , "compiler/objectivec/enum.cc" , "compiler/objectivec/enum_field.cc" , "compiler/objectivec/extension.cc" , "compiler/objectivec/field.cc" , "compiler/objectivec/file.cc" , "compiler/objectivec/generator.cc" , "compiler/objectivec/helpers.cc" , "compiler/objectivec/import_writer.cc" , "compiler/objectivec/line_consumer.cc" , "compiler/objectivec/map_field.cc" , "compiler/objectivec/message.cc" , "compiler/objectivec/message_field.cc" , "compiler/objectivec/names.cc" , "compiler/objectivec/oneof.cc" , "compiler/objectivec/primitive_field.cc" , "compiler/objectivec/tf_decode_data.cc" , "compiler/php/names.cc" , "compiler/php/php_generator.cc" , "compiler/plugin.cc" , "compiler/plugin.pb.cc" , "compiler/python/generator.cc" , "compiler/python/helpers.cc" , "compiler/python/pyi_generator.cc" , "compiler/retention.cc" , "compiler/ruby/ruby_generator.cc" , "compiler/rust/accessors/accessor_case.cc" , "compiler/rust/accessors/accessors.cc" , "compiler/rust/accessors/default_value.cc" , "compiler/rust/accessors/map.cc" , "compiler/rust/accessors/repeated_field.cc" , "compiler/rust/accessors/singular_cord.cc" , "compiler/rust/accessors/singular_message.cc" , "compiler/rust/accessors/singular_scalar.cc" , "compiler/rust/accessors/singular_string.cc" , "compiler/rust/accessors/unsupported_field.cc" , "compiler/rust/accessors/with_presence.cc" , "compiler/rust/context.cc" , "compiler/rust/crate_mapping.cc" , "compiler/rust/enum.cc" , "compiler/rust/generator.cc" , "compiler/rust/message.cc" , "compiler/rust/naming.cc" , "compiler/rust/oneof.cc" , "compiler/rust/relative_path.cc" , "compiler/rust/rust_field_type.cc" , "compiler/rust/rust_keywords.cc" , "compiler/rust/upb_helpers.cc" , "compiler/subprocess.cc" , "compiler/versions.cc" , "compiler/zip_writer.cc" , "testing/file.cc" , ["upb_generator", "names_srcs"] ] , "deps": [["@", "absl", "absl/log", "initialize"], ["", "libprotobuf"]] } , "protoc": { "type": "configure" , "target": "protoc (unconfigured)" , "config": {"type": "'", "$1": {"DEBUG": null}} } , "protoc (unconfigured)": { "type": ["@", "rules", "CC", "binary"] , "name": ["protoc"] , "private-deps": [["", "libprotoc"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/include/000077500000000000000000000000001516554100600241165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/include/openssl/000077500000000000000000000000001516554100600256015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/include/openssl/TARGETS.boringssl000066400000000000000000000033061516554100600306400ustar00rootroot00000000000000{ "crypto_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "aead.h" , "aes.h" , "arm_arch.h" , "asm_base.h" , "asn1.h" , "asn1_mac.h" , "asn1t.h" , "base.h" , "base64.h" , "bcm_public.h" , "bio.h" , "blake2.h" , "blowfish.h" , "bn.h" , "buf.h" , "buffer.h" , "bytestring.h" , "cast.h" , "chacha.h" , "cipher.h" , "cmac.h" , "conf.h" , "cpu.h" , "crypto.h" , "ctrdrbg.h" , "curve25519.h" , "des.h" , "dh.h" , "digest.h" , "dsa.h" , "e_os2.h" , "ec.h" , "ec_key.h" , "ecdh.h" , "ecdsa.h" , "engine.h" , "err.h" , "evp.h" , "evp_errors.h" , "ex_data.h" , "experimental/kyber.h" , "hkdf.h" , "hmac.h" , "hpke.h" , "hrss.h" , "is_boringssl.h" , "kdf.h" , "lhash.h" , "md4.h" , "md5.h" , "mem.h" , "mldsa.h" , "mlkem.h" , "nid.h" , "obj.h" , "obj_mac.h" , "objects.h" , "opensslconf.h" , "opensslv.h" , "ossl_typ.h" , "pem.h" , "pkcs12.h" , "pkcs7.h" , "pkcs8.h" , "poly1305.h" , "pool.h" , "posix_time.h" , "rand.h" , "rc4.h" , "ripemd.h" , "rsa.h" , "safestack.h" , "service_indicator.h" , "sha.h" , "siphash.h" , "slhdsa.h" , "span.h" , "stack.h" , "target.h" , "thread.h" , "time.h" , "trust_token.h" , "type_check.h" , "x509.h" , "x509_vfy.h" , "x509v3.h" , "x509v3_errors.h" ] , "stage": ["openssl"] } , "ssl_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["dtls1.h", "srtp.h", "ssl.h", "ssl3.h", "tls1.h"] , "stage": ["openssl"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/lib/000077500000000000000000000000001516554100600232415ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/lib/TARGETS.cares000066400000000000000000000056771516554100600254100ustar00rootroot00000000000000{ "ares_lib": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS"] , "name": ["libcares"] , "pure C": ["YES"] , "private-defines": { "type": "++" , "$1": [ ["HAVE_CONFIG_H", "CARES_STATICLIB"] , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "darwin": ["_DARWIN_C_SOURCE"] , "linux": ["_GNU_SOURCE", "_POSIX_C_SOURCE=199309L", "_XOPEN_SOURCE=600"] , "sunos": ["__EXTENSIONS__", "_REENTRANT", "_XOPEN_SOURCE=600"] , "aix": ["_ALL_SOURCE", "_XOPEN_SOURCE=600", "_USE_IRS"] , "windows": [ "WIN32_LEAN_AND_MEAN" , "_CRT_SECURE_NO_DEPRECATE" , "_CRT_NONSTDC_NO_DEPRECATE" , "_WIN32_WINNT=0x0600" ] } } ] } , "srcs": [ "ares_android.c" , "ares_cancel.c" , "ares_create_query.c" , "ares_data.c" , "ares_destroy.c" , "ares_expand_name.c" , "ares_expand_string.c" , "ares_fds.c" , "ares_freeaddrinfo.c" , "ares_free_hostent.c" , "ares_free_string.c" , "ares_getaddrinfo.c" , "ares_getenv.c" , "ares_gethostbyaddr.c" , "ares_gethostbyname.c" , "ares_getnameinfo.c" , "ares_getsock.c" , "ares_init.c" , "ares_library_init.c" , "ares_llist.c" , "ares_mkquery.c" , "ares_nowarn.c" , "ares_options.c" , "ares_parse_aaaa_reply.c" , "ares_parse_a_reply.c" , "ares_parse_caa_reply.c" , "ares_parse_mx_reply.c" , "ares_parse_naptr_reply.c" , "ares_parse_ns_reply.c" , "ares_parse_ptr_reply.c" , "ares_parse_soa_reply.c" , "ares_parse_srv_reply.c" , "ares_parse_txt_reply.c" , "ares_parse_uri_reply.c" , "ares_platform.c" , "ares_process.c" , "ares_query.c" , "ares_rand.c" , "ares_search.c" , "ares_send.c" , "ares_strcasecmp.c" , "ares_strdup.c" , "ares_strerror.c" , "ares_strsplit.c" , "ares_timeout.c" , "ares_version.c" , "ares_writev.c" , "ares__addrinfo2hostent.c" , "ares__addrinfo_localhost.c" , "ares__close_sockets.c" , "ares__get_hostent.c" , "ares__parse_into_addrinfo.c" , "ares__readaddrinfo.c" , "ares__read_line.c" , "ares__sortaddrinfo.c" , "ares__timeval.c" , "bitncmp.c" , "inet_net_pton.c" , "inet_ntop.c" , "windows_port.c" ] , "hdrs": [ ["@", "grpc", "third_party", "ares_build_h"] , ["@", "grpc", "third_party", "ares_config_h"] , ["include", "ares_include_headers"] , "ares_android.h" , "ares_data.h" , "ares_getenv.h" , "ares_inet_net_pton.h" , "ares_iphlpapi.h" , "ares_ipv6.h" , "ares_llist.h" , "ares_nowarn.h" , "ares_platform.h" , "ares_private.h" , "ares_setup.h" , "ares_strcasecmp.h" , "ares_strdup.h" , "ares_strsplit.h" , "ares_writev.h" , "bitncmp.h" , "config-dos.h" , "config-win32.h" , "setup_once.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/000077500000000000000000000000001516554100600240275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/TARGETS.git2000066400000000000000000000014541516554100600257330ustar00rootroot00000000000000{ "libgit2_private_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ ["GLOB", null, "*.h"] , ["./", "streams", "hdrs"] , ["./", "transports", "hdrs"] , ["src/util", "util_private_headers"] , "experimental.h" ] } , "libgit2_sources": { "type": ["@", "rules", "data", "staged"] , "srcs": [ ["GLOB", null, "*.c"] , ["./", "streams", "srcs"] , ["./", "transports", "srcs"] ] } , "experimental.h": { "type": "configure" , "target": "experimental" , "config": { "type": "let*" , "bindings": [["defines1", [["GIT_EXPERIMENTAL_SHA256", 0]]]] , "body": {"type": "env", "vars": []} } } , "experimental": { "type": ["@", "rules", "CC/auto", "config"] , "name": ["experimental.h"] , "guard": ["INCLUDE_experimental_h__"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/streams/000077500000000000000000000000001516554100600255055ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/streams/TARGETS.git2000066400000000000000000000003621516554100600274060ustar00rootroot00000000000000{ "srcs": { "type": ["@", "rules", "data", "staged"] , "stage": ["streams"] , "srcs": [["GLOB", null, "*.c"]] } , "hdrs": { "type": ["@", "rules", "data", "staged"] , "stage": ["streams"] , "srcs": [["GLOB", null, "*.h"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/transports/000077500000000000000000000000001516554100600262465ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/libgit2/transports/TARGETS.git2000066400000000000000000000003701516554100600301460ustar00rootroot00000000000000{ "srcs": { "type": ["@", "rules", "data", "staged"] , "stage": ["transports"] , "srcs": [["GLOB", null, "*.c"]] } , "hdrs": { "type": ["@", "rules", "data", "staged"] , "stage": ["transports"] , "srcs": [["GLOB", null, "*.h"]] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/000077500000000000000000000000001516554100600241255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/api/000077500000000000000000000000001516554100600246765ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/api/TARGETS.lzma000066400000000000000000000006111516554100600266720ustar00rootroot00000000000000{ "public_headers": { "type": "install" , "deps": [ "lzma.h" , "lzma/base.h" , "lzma/bcj.h" , "lzma/block.h" , "lzma/check.h" , "lzma/container.h" , "lzma/delta.h" , "lzma/filter.h" , "lzma/hardware.h" , "lzma/index.h" , "lzma/index_hash.h" , "lzma/lzma12.h" , "lzma/stream_flags.h" , "lzma/version.h" , "lzma/vli.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/check/000077500000000000000000000000001516554100600252025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/check/TARGETS.lzma000066400000000000000000000012501516554100600271760ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ENABLE_SMALL", "ADDITIONAL_CHECK_TYPES"] , "hdrs": { "type": "++" , "$1": [ ["check.h", "crc_common.h"] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_SMALL"} , "then": [] , "else": { "type": "++" , "$1": [ ["crc32_table_be.h", "crc32_table_le.h", "crc_x86_clmul.h"] , { "type": "if" , "cond": {"type": "var", "name": "ADDITIONAL_CHECK_TYPES"} , "then": ["crc64_table_be.h", "crc64_table_le.h"] } ] } } ] } } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/common/000077500000000000000000000000001516554100600254155ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/common/TARGETS.lzma000066400000000000000000000033161516554100600274160ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ENABLE_THREADS", "ENCODERS", "DECODERS", "LZIP_DECODER"] , "hdrs": { "type": "++" , "$1": [ [ "common.h" , "easy_preset.h" , "filter_common.h" , "index.h" , "memcmplen.h" , "stream_flags_common.h" ] , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_THREADS"} , "then": ["outqueue.h"] } , { "type": "if" , "cond": {"type": "var", "name": "LZIP_DECODER"} , "then": ["lzip_decoder.h"] } ] } , "deps": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "ENCODERS"} , "then": ["encoder_headers"] } , { "type": "if" , "cond": {"type": "var", "name": "DECODERS"} , "then": ["decoder_headers"] } ] } } , "encoder_headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": [ "block_buffer_encoder.h" , "block_encoder.h" , "filter_encoder.h" , "index_encoder.h" ] , "deps": [ ["src/liblzma/delta", "headers"] , ["src/liblzma/lzma", "lzma1_headers"] , ["src/liblzma/lzma", "lzma2_headers"] , ["src/liblzma/simple", "headers"] ] } , "decoder_headers": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["LZIP_DECODER"] , "hdrs": [ "alone_decoder.h" , "block_decoder.h" , "filter_decoder.h" , "index_decoder.h" , "stream_decoder.h" ] , "deps": [ ["src/liblzma/delta", "headers"] , ["src/liblzma/lzma", "lzma1_headers"] , ["src/liblzma/lzma", "lzma2_headers"] , ["src/liblzma/simple", "headers"] ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/delta/000077500000000000000000000000001516554100600252165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/delta/TARGETS.lzma000066400000000000000000000002331516554100600272120ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": ["delta_encoder.h", "delta_decoder.h", "delta_common.h", "delta_private.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/lz/000077500000000000000000000000001516554100600245525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/lz/TARGETS.lzma000066400000000000000000000002651516554100600265530ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": [ "lz_encoder.h" , "lz_encoder_hash.h" , "lz_encoder_hash_table.h" , "lz_decoder.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/lzma/000077500000000000000000000000001516554100600250705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/lzma/TARGETS.lzma000066400000000000000000000006311516554100600270660ustar00rootroot00000000000000{ "lzma1_headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": [ "fastpos.h" , "lzma_common.h" , "lzma_encoder.h" , "lzma_encoder_private.h" , "lzma_decoder.h" ] , "deps": [["src/liblzma/lz", "headers"], ["src/liblzma/rangecoder", "headers"]] } , "lzma2_headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": ["lzma2_encoder.h", "lzma2_decoder.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/rangecoder/000077500000000000000000000000001516554100600262365ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/rangecoder/TARGETS.lzma000066400000000000000000000002171516554100600302340ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": ["price.h", "range_common.h", "range_encoder.h", "range_decoder.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/simple/000077500000000000000000000000001516554100600254165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/liblzma/simple/TARGETS.lzma000066400000000000000000000002631516554100600274150ustar00rootroot00000000000000{ "headers": { "type": ["@", "rules", "CC", "library"] , "hdrs": [ "simple_encoder.h" , "simple_decoder.h" , "simple_coder.h" , "simple_private.h" ] } } just-buildsystem-justbuild-b1fb5fa/etc/import/src/util/000077500000000000000000000000001516554100600234505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/src/util/TARGETS.git2000066400000000000000000000440201516554100600253500ustar00rootroot00000000000000{ "util_private_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "alloc.h" , "array.h" , "assert_safe.h" , "bitvec.h" , "cc-compat.h" , "date.h" , "filebuf.h" , "fs_path.h" , "futils.h" , "git2_util.h" , "hash.h" , "integer.h" , "khash.h" , "map.h" , "net.h" , "pool.h" , "posix.h" , "pqueue.h" , "rand.h" , "regexp.h" , "runtime.h" , "sortedcache.h" , "staticstr.h" , "str.h" , "strmap.h" , "strnlen.h" , "thread.h" , "utf8.h" , "util.h" , "varint.h" , "vector.h" , "wildmatch.h" , "zstream.h" , "allocators/failalloc.h" , "allocators/stdalloc.h" , "allocators/win32_leakcheck.h" , "hash/builtin.h" , "hash/collisiondetect.h" , "hash/common_crypto.h" , "hash/mbedtls.h" , "hash/openssl.h" , "hash/sha.h" , "hash/win32.h" , "hash/sha1dc/sha1.h" , "hash/sha1dc/ubc_check.h" , "hash/rfc6234/sha.h" , "unix/posix.h" , "unix/pthread.h" , "win32/dir.h" , "win32/error.h" , "win32/mingw-compat.h" , "win32/msvc-compat.h" , "win32/path_w32.h" , "win32/posix.h" , "win32/precompiled.h" , "win32/reparse.h" , "win32/thread.h" , "win32/utf-conv.h" , "win32/version.h" , "win32/w32_buffer.h" , "win32/w32_common.h" , "win32/w32_leakcheck.h" , "win32/w32_util.h" , "win32/win32-compat.h" , ["include", "git2_public_headers"] , "git2_features.h" , ["deps/xdiff", "hdrs"] ] } , "util_os_unix": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "USE_THREADS"] , "name": ["util_os_unix"] , "pure C": ["yes"] , "srcs": ["unix/map.c", "unix/realpath.c"] , "private-hdrs": ["util_private_headers"] , "private-ldflags": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "USE_THREADS"} , "then": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": { "solaris": ["-lsocket", "-lnsl"] , "sunos": ["-lsocket", "-lnsl"] , "haiku": ["-lnetwork"] } } ] } } , "util_os_win32": { "type": ["@", "rules", "CC", "library"] , "name": ["util_os_win32"] , "pure C": ["yes"] , "srcs": [ "win32/dir.c" , "win32/error.c" , "win32/map.c" , "win32/path_w32.c" , "win32/posix_w32.c" , "win32/precompiled.c" , "win32/thread.c" , "win32/utf-conv.c" , "win32/w32_buffer.c" , "win32/w32_leakcheck.c" , "win32/w32_util.c" ] , "private-hdrs": ["util_private_headers"] , "private-ldflags": ["-lws2_32"] } , "util_hash_collision_detection": { "type": ["@", "rules", "CC", "library"] , "name": ["util_hash_collision_detection"] , "pure C": ["yes"] , "private-defines": [ "SHA1DC_NO_STANDARD_INCLUDES=1" , "SHA1DC_CUSTOM_INCLUDE_SHA1_C=\"git2_util.h\"" , "SHA1DC_CUSTOM_INCLUDE_UBC_CHECK_C=\"git2_util.h\"" ] , "srcs": ["hash/collisiondetect.c", "hash/sha1dc/sha1.c", "hash/sha1dc/ubc_check.c"] , "private-hdrs": ["util_private_headers"] } , "util_hash_openssl": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS"] , "name": ["util_hash_openssl"] , "pure C": ["yes"] , "srcs": ["hash/openssl.c"] , "private-hdrs": ["util_private_headers"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lssl"] } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ssl", "", "crypto"]] } } , "util_hash_common_crypto": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS"] , "name": ["util_hash_common_crypto"] , "pure C": ["yes"] , "srcs": ["hash/common_crypto.c"] , "private-hdrs": ["util_private_headers"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lcommon_crypto"] } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "common_crypto", "", "crypto"]] } } , "util_hash_mbedtls": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS"] , "name": ["util_hash_mbedtls"] , "pure C": ["yes"] , "srcs": ["hash/mbedtls.c"] , "private-hdrs": ["util_private_headers"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lmbedtls"] } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "mbedtls", "", "crypto"]] } } , "util_hash_win32": { "type": ["@", "rules", "CC", "library"] , "name": ["util_hash_win32"] , "pure C": ["yes"] , "srcs": ["hash/win32.c"] , "private-hdrs": ["util_private_headers"] } , "util_hash_builtin": { "type": ["@", "rules", "CC", "library"] , "name": ["util_hash_builtin"] , "pure C": ["yes"] , "srcs": ["hash/builtin.c", "hash/rfc6234/sha224-256.c"] , "private-hdrs": ["util_private_headers"] } , "util_os": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS"] , "deps": { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "OS"}, "$2": "windows"} , "then": ["util_os_win32"] , "else": ["util_os_unix"] } } , "util_sha1": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SHA1"] , "deps": { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA1"} , "case": [ ["CollisionDetection", ["util_hash_collision_detection"]] , ["OpenSSL", ["util_hash_openssl"]] , ["CommonCrypto", ["util_hash_common_crypto"]] , ["mbedTLS", ["util_hash_mbedtls"]] , ["Win32", ["util_hash_win32"]] , ["Generic", ["util_hash_generic"]] ] , "default": {"type": "fail", "msg": "Asked for unknown SHA1 backend in `USE_SHA1`"} } } , "util_sha256": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SHA256"] , "deps": { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA256"} , "case": [ ["Builtin", ["util_hash_builtin"]] , ["OpenSSL", ["util_hash_openssl"]] , ["CommonCrypto", ["util_hash_common_crypto"]] , ["mbedTLS", ["util_hash_mbedtls"]] , ["Win32", ["util_hash_win32"]] ] , "default": { "type": "fail" , "msg": "Asked for unknown SHA256 backend in `USE_SHA256`" } } } , "util_regex": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "REGEX_BACKEND"] , "name": ["util_regex"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case" , "expr": {"type": "var", "name": "REGEX_BACKEND"} , "case": {"pcre2": ["-lpcre2-8"], "pcre": ["-lpcre"]} } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "REGEX_BACKEND"} , "case": [ ["builtin", [["deps/pcre", "git2_pcre"]]] , [null, [["deps/pcre", "git2_pcre"]]] ] } , "else": { "type": "case*" , "expr": {"type": "var", "name": "REGEX_BACKEND"} , "case": [ ["pcre2", [["@", "pcre2", "", "pcre2"]]] , ["pcre", [["@", "pcre", "", "pcre"]]] , ["builtin", [["deps/pcre", "git2_pcre"]]] , [null, [["deps/pcre", "git2_pcre"]]] ] } } } , "util_compress": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_BUNDLED_ZLIB"] , "name": ["util_compress"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_BUNDLED_ZLIB"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lz"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_BUNDLED_ZLIB"} , "then": {"type": "fail", "msg": "bundled zlib from deps/zlib not supported yet."} , "else": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "zlib", "", "zlib"]] } } } , "util_ssh": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_SSH"] , "name": ["util_ssh"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SSH"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lssh2"] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SSH"} , "then": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "ssh2", "", "ssh"]] } } } , "util_http_parser": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "USE_HTTP_PARSER"] , "name": ["util_http_parser"] , "private-ldflags": { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTP_PARSER"} , "case": [ [ "system" , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": ["-lhttp_parser"] } ] ] } , "deps": { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTP_PARSER"} , "case": [ [ "system" , { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": [["@", "http_parser", "", "http_parser"]] } ] ] , "default": [["deps/http-parser", "git2_http_parser"]] } } , "util_gssapi": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "USE_SYSTEM_LIBS", "USE_GSSAPI"] , "name": ["util_gssapi"] , "private-ldflags": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "USE_GSSAPI"} , "case": [ [ true , { "type": "case*" , "expr": {"type": "var", "name": "OS"} , "case": [["darwin", ["-framework", "GSS"]]] , "default": ["-lgssapi"] } ] , ["gssapi", ["-lgssapi"]] , ["GSS.framework", ["-framework", "GSS"]] ] } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "if" , "cond": {"type": "var", "name": "USE_GSSAPI"} , "then": [["@", "gssapi", "", "gssapi"]] } } } , "util_https": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["OS", "USE_SYSTEM_LIBS", "USE_HTTPS"] , "name": ["util_https"] , "private-ldflags": { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [["WinHTTP", ["-lwinhttp"]]] , "default": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ ["OpenSSL", ["-lssl"]] , ["mbedTLS", ["-lmbedtls"]] , [ "SecureTransport" , ["-framework", "CoreFoundation", "-framework", "Security"] ] ] } } } , "deps": { "type": "if" , "cond": {"type": "var", "name": "USE_SYSTEM_LIBS"} , "then": [] , "else": { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ ["OpenSSL", [["@", "ssl", "", "ssl"]]] , ["mbedTLS", [["@", "mbedtls", "", "ssl"]]] , ["SecureTransport", [["@", "secure_transport", "", "ssl"]]] ] } } } , "util_sources": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "alloc.c" , "date.c" , "filebuf.c" , "fs_path.c" , "futils.c" , "hash.c" , "net.c" , "pool.c" , "posix.c" , "pqueue.c" , "rand.c" , "regexp.c" , "runtime.c" , "sortedcache.c" , "str.c" , "strmap.c" , "thread.c" , "tsort.c" , "utf8.c" , "util.c" , "varint.c" , "vector.c" , "wildmatch.c" , "zstream.c" , "allocators/failalloc.c" , "allocators/stdalloc.c" , "allocators/win32_leakcheck.c" , ["deps/xdiff", "srcs"] ] } , "git2_features.h": { "type": "configure" , "arguments_config": [ "OS" , "ARCH" , "TARGET_ARCH" , "DEBUG_POOL" , "USE_THREADS" , "REGEX_BACKEND" , "USE_ICONV" , "USE_NSEC" , "USE_SSH" , "USE_NTLMCLIENT" , "USE_GSSAPI" , "USE_SHA1" , "USE_SHA256" , "USE_HTTPS" ] , "target": "feature_header" , "config": { "type": "let*" , "bindings": [ [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] , [ "IS_BSD" , { "type": "case" , "expr": {"type": "var", "name": "OS"} , "case": {"darwin": true, "bsd": true} } ] , [ "IS_32BIT" , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": {"x86": true, "arm": true} } ] , [ "defines1" , { "type": "++" , "$1": [ [ ["GIT_DEBUG_POOL", {"type": "var", "name": "DEBUG_POOL"}] , ["GIT_THREADS", {"type": "var", "name": "USE_THREADS"}] , { "type": "if" , "cond": {"type": "var", "name": "IS_32BIT"} , "then": ["GIT_ARCH_32", 1] , "else": ["GIT_ARCH_64", 1] } , ["GIT_USE_ICONV", {"type": "var", "name": "USE_ICONV"}] , ["GIT_SSH", {"type": "var", "name": "USE_SSH"}] , [ "GIT_SSH_MEMORY_CREDENTIALS" , {"type": "var", "name": "USE_SSH"} ] , ["GIT_NTLM", {"type": "var", "name": "USE_NTLMCLIENT"}] , [ "GIT_IO_WSAPOLL" , { "type": "==" , "$1": {"type": "var", "name": "OS"} , "$2": "windows" } ] ] , { "type": "if" , "cond": {"type": "var", "name": "USE_NSEC"} , "then": [ ["GIT_USE_NSEC", 1] , ["GIT_USE_FUTIMENS", 1] , ["GIT_USE_STAT_MTIME_NSEC", 1] , { "type": "if" , "cond": {"type": "var", "name": "IS_BSD"} , "then": ["GIT_USE_STAT_MTIMESPEC", 1] , "else": ["GIT_USE_STAT_MTIM", 1] } ] } , [ { "type": "case*" , "expr": {"type": "var", "name": "REGEX_BACKEND"} , "case": [ ["regcomp_l", ["GIT_REGEX_REGCOMP_L", 1]] , ["regcomp", ["GIT_REGEX_REGCOMP", 1]] , ["pcre", ["GIT_REGEX_PCRE", 1]] , ["pcre2", ["GIT_REGEX_PCRE2", 1]] , ["builtin", ["GIT_REGEX_BUILTIN", 1]] , [null, ["GIT_REGEX_BUILTIN", 1]] ] , "default": { "type": "fail" , "msg": "The REGEX_BACKEND option provided is not supported" } } ] , { "type": "case*" , "expr": {"type": "var", "name": "USE_GSSAPI"} , "case": [ ["GSS.framework", [["GIT_GSSFRAMEWORK", 1]]] , ["gssapi", [["GIT_GSSAPI", 1]]] , [false, []] ] , "default": { "type": "fail" , "msg": "Backend asked for in USE_GSSAPI is not supported" } } , { "type": "if" , "cond": {"type": "var", "name": "USE_HTTPS"} , "then": [ ["GIT_HTTPS", 1] , { "type": "case*" , "expr": {"type": "var", "name": "USE_HTTPS"} , "case": [ ["SecureTransport", ["GIT_SECURE_TRANSPORT", 1]] , ["OpenSSL", ["GIT_OPENSSL", 1]] , ["mbedTLS", ["GIT_MBEDTLS", 1]] , ["WinHTTP", ["GIT_WINHTTP", 1]] ] , "default": { "type": "fail" , "msg": "Backend asked for in USE_HTTPS is not supported" } } ] } , { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA1"} , "case": [ ["CollisionDetection", [["GIT_SHA1_COLLISIONDETECT", 1]]] , ["Win32", [["GIT_SHA1_WIN32", 1]]] , ["CommonCrypto", [["GIT_SHA1_COMMON_CRYPTO", 1]]] , ["OpenSSL", [["GIT_SHA1_OPENSSL", 1]]] , ["mbedTLS", [["GIT_SHA1_MBEDTLS", 1]]] ] } , { "type": "case*" , "expr": {"type": "var", "name": "USE_SHA256"} , "case": [ ["Builtin", [["GIT_SHA256_BUILTIN", 1]]] , ["Win32", [["GIT_SHA256_WIN32", 1]]] , ["CommonCrypto", [["GIT_SHA256_COMMON_CRYPTO", 1]]] , ["OpenSSL", [["GIT_SHA256_OPENSSL", 1]]] , ["mbedTLS", [["GIT_SHA256_MBEDTLS", 1]]] ] } ] } ] , [ "have_csymbol" , [ ["GIT_IO_POLL", ["poll", ["poll.h"]]] , ["GIT_IO_SELECT", ["select", ["sys/select.h"]]] ] ] ] , "body": {"type": "env", "vars": ["defines1", "have_csymbol"]} } } , "feature_header": { "type": ["@", "rules", "CC/auto", "config"] , "name": ["git2_features.h"] , "guard": ["INCLUDE_features_h__"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/000077500000000000000000000000001516554100600242355ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/TARGETS.grpc000066400000000000000000000031421516554100600262230ustar00rootroot00000000000000{ "ares_build_h": {"type": "install", "files": {"ares_build.h": "cares/ares_build.h"}} , "ares_config_h": { "type": "install" , "arguments_config": ["OS", "ARCH", "TARGET_ARCH"] , "files": { "ares_config.h": { "type": "case" , "expr": { "type": "var" , "name": "OS" , "default": {"type": "fail", "msg": "Required variable 'OS' is not set."} } , "case": { "ios": "cares/config_darwin/ares_config.h" , "tvos": "cares/config_darwin/ares_config.h" , "watchos": "cares/config_darwin/ares_config.h" , "darwin": "cares/config_darwin/ares_config.h" , "windows": "cares/config_windows/ares_config.h" , "android": "cares/config_android/ares_config.h" } , "default": "cares/config_linux/ares_config.h" } } } , "address_sorting_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": ["address_sorting/address_sorting_internal.h"] , "stage": ["third_party", "address_sorting"] , "deps": [ [ "./" , "address_sorting/include/address_sorting" , "address_sorting_headers" ] ] } , "address_sorting_srcs": { "type": ["@", "rules", "data", "staged"] , "stage": ["third_party", "address_sorting"] , "srcs": [ "address_sorting/address_sorting.c" , "address_sorting/address_sorting_posix.c" , "address_sorting/address_sorting_windows.c" ] } , "address_sorting": { "type": ["@", "rules", "CC", "library"] , "name": ["address_sorting"] , "pure C": ["YES"] , "srcs": ["address_sorting_srcs"] , "hdrs": ["address_sorting_headers"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/address_sorting/000077500000000000000000000000001516554100600274275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/address_sorting/TARGETS.grpc000066400000000000000000000005431516554100600314170ustar00rootroot00000000000000{ "address_sorting": { "type": ["@", "rules", "CC", "library"] , "name": ["address_sorting"] , "srcs": [ "address_sorting.c" , "address_sorting_posix.c" , "address_sorting_windows.c" ] , "hdrs": [ "address_sorting_internal.h" , ["./", "include/address_sorting", "address_sorting_headers"] ] , "pure C": ["YES"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/address_sorting/include/000077500000000000000000000000001516554100600310525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/address_sorting/include/address_sorting/000077500000000000000000000000001516554100600342445ustar00rootroot00000000000000TARGETS.grpc000066400000000000000000000002231516554100600361500ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/address_sorting/include/address_sorting{ "address_sorting_headers": { "type": ["@", "rules", "data", "staged"] , "stage": ["address_sorting"] , "srcs": ["address_sorting.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/upb/000077500000000000000000000000001516554100600250235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/upb/TARGETS.grpc000066400000000000000000000337031516554100600270170ustar00rootroot00000000000000{ "generated_code_support": { "type": ["@", "rules", "CC", "library"] , "name": ["generated_code_support"] , "hdrs": ["upb/generated_code_support.h", "upb/port/def.inc", "upb/port/undef.inc"] , "deps": [ "base" , "mem" , "message" , "message_internal" , "mini_descriptor" , "mini_table" , "wire" ] } , "reflection": { "type": ["@", "rules", "CC", "library"] , "name": ["reflection"] , "hdrs": [ "upb/reflection/def.h" , "upb/reflection/def.hpp" , "upb/reflection/message.h" , "upb/reflection/message.hpp" ] , "deps": [ "base" , "mem" , "message" , "message_internal" , "mini_descriptor" , "mini_table" , "port" , "reflection_internal" ] , "pure C": ["YES"] } , "reflection_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["reflection_internal"] , "srcs": [ "upb/reflection/def_pool.c" , "upb/reflection/def_type.c" , "upb/reflection/desc_state.c" , "upb/reflection/enum_def.c" , "upb/reflection/enum_reserved_range.c" , "upb/reflection/enum_value_def.c" , "upb/reflection/extension_range.c" , "upb/reflection/field_def.c" , "upb/reflection/file_def.c" , "upb/reflection/internal/def_builder.c" , "upb/reflection/internal/strdup2.c" , "upb/reflection/message.c" , "upb/reflection/message_def.c" , "upb/reflection/message_reserved_range.c" , "upb/reflection/method_def.c" , "upb/reflection/oneof_def.c" , "upb/reflection/service_def.c" ] , "hdrs": [ "upb/reflection/common.h" , "upb/reflection/def.h" , "upb/reflection/def.hpp" , "upb/reflection/def_pool.h" , "upb/reflection/def_type.h" , "upb/reflection/enum_def.h" , "upb/reflection/enum_reserved_range.h" , "upb/reflection/enum_value_def.h" , "upb/reflection/extension_range.h" , "upb/reflection/field_def.h" , "upb/reflection/file_def.h" , "upb/reflection/internal/def_pool.h" , "upb/reflection/internal/desc_state.h" , "upb/reflection/internal/enum_def.h" , "upb/reflection/internal/enum_reserved_range.h" , "upb/reflection/internal/enum_value_def.h" , "upb/reflection/internal/extension_range.h" , "upb/reflection/internal/field_def.h" , "upb/reflection/internal/file_def.h" , "upb/reflection/internal/message_def.h" , "upb/reflection/internal/message_reserved_range.h" , "upb/reflection/internal/method_def.h" , "upb/reflection/internal/oneof_def.h" , "upb/reflection/internal/service_def.h" , "upb/reflection/internal/upb_edition_defaults.h" , "upb/reflection/message.h" , "upb/reflection/message.hpp" , "upb/reflection/message_def.h" , "upb/reflection/message_reserved_range.h" , "upb/reflection/method_def.h" , "upb/reflection/oneof_def.h" , "upb/reflection/service_def.h" , "upb/reflection/descriptor_bootstrap.h" ] , "private-hdrs": [ "upb/reflection/internal/def_builder.h" , "upb/reflection/internal/strdup2.h" ] , "deps": [ "base" , "base_internal" , "hash" , "mem" , "message" , "message_copy" , "message_internal" , "mini_descriptor" , "mini_descriptor_internal" , "mini_table" , "mini_table_internal" , "port" , "wire" , ["src/core/ext/upb-gen", "upb-gen-lib"] ] , "pure C": ["YES"] } , "base": { "type": ["@", "rules", "CC", "library"] , "name": ["base"] , "srcs": ["upb/base/status.c"] , "hdrs": [ "upb/base/descriptor_constants.h" , "upb/base/status.h" , "upb/base/status.hpp" , "upb/base/string_view.h" , "upb/base/upcast.h" ] , "deps": ["port"] , "pure C": ["YES"] } , "base_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["base_internal"] , "hdrs": ["upb/base/internal/endian.h", "upb/base/internal/log2.h"] , "deps": ["port"] , "pure C": ["YES"] } , "hash": { "type": ["@", "rules", "CC", "library"] , "name": ["hash"] , "srcs": ["upb/hash/common.c"] , "hdrs": ["upb/hash/common.h", "upb/hash/int_table.h", "upb/hash/str_table.h"] , "deps": ["base", "base_internal", "mem", "port"] , "pure C": ["YES"] } , "string": { "type": ["@", "rules", "CC", "library"] , "name": ["string"] , "hdrs": ["upb/io/string.h"] , "deps": ["mem", "port"] , "pure C": ["YES"] } , "tokenizer": { "type": ["@", "rules", "CC", "library"] , "name": ["tokenizer"] , "srcs": ["upb/io/tokenizer.c"] , "hdrs": ["upb/io/tokenizer.h"] , "deps": ["base", "lex", "mem", "port", "string", "zero_copy_stream"] , "pure C": ["YES"] } , "zero_copy_stream": { "type": ["@", "rules", "CC", "library"] , "name": ["zero_copy_stream"] , "hdrs": ["upb/io/zero_copy_input_stream.h", "upb/io/zero_copy_output_stream.h"] , "deps": ["base", "mem", "port"] , "pure C": ["YES"] } , "json": { "type": ["@", "rules", "CC", "library"] , "name": ["json"] , "srcs": ["upb/json/decode.c", "upb/json/encode.c"] , "hdrs": ["upb/json/decode.h", "upb/json/encode.h"] , "deps": [ "base" , "lex" , "mem" , "message" , "mini_table" , "port" , "reflection" , "wire" ] , "pure C": ["YES"] } , "lex": { "type": ["@", "rules", "CC", "library"] , "name": ["lex"] , "srcs": [ "upb/lex/atoi.c" , "upb/lex/round_trip.c" , "upb/lex/strtod.c" , "upb/lex/unicode.c" ] , "hdrs": [ "upb/lex/atoi.h" , "upb/lex/round_trip.h" , "upb/lex/strtod.h" , "upb/lex/unicode.h" ] , "deps": ["port"] , "pure C": ["YES"] } , "mem": { "type": ["@", "rules", "CC", "library"] , "name": ["mem"] , "srcs": ["upb/mem/alloc.c", "upb/mem/arena.c"] , "hdrs": ["upb/mem/alloc.h", "upb/mem/arena.h", "upb/mem/arena.hpp"] , "deps": ["mem_internal", "port"] , "pure C": ["YES"] } , "mem_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["mem_internal"] , "hdrs": ["upb/mem/internal/arena.h"] , "deps": ["port"] , "pure C": ["YES"] } , "message": { "type": ["@", "rules", "CC", "library"] , "name": ["message"] , "srcs": [ "upb/message/accessors.c" , "upb/message/array.c" , "upb/message/compat.c" , "upb/message/map.c" , "upb/message/map_sorter.c" , "upb/message/message.c" ] , "hdrs": [ "upb/message/accessors.h" , "upb/message/array.h" , "upb/message/compat.h" , "upb/message/map.h" , "upb/message/map_gencode_util.h" , "upb/message/message.h" , "upb/message/tagged_ptr.h" , "upb/message/value.h" ] , "deps": [ "base" , "base_internal" , "hash" , "mem" , "message_internal" , "message_types" , "mini_table" , "mini_table_internal" , "port" ] , "pure C": ["YES"] } , "message_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["message_internal"] , "srcs": [ "upb/message/internal/compare_unknown.c" , "upb/message/internal/extension.c" , "upb/message/internal/message.c" ] , "hdrs": [ "upb/message/internal/accessors.h" , "upb/message/internal/array.h" , "upb/message/internal/compare_unknown.h" , "upb/message/internal/extension.h" , "upb/message/internal/map.h" , "upb/message/internal/map_sorter.h" , "upb/message/internal/message.h" , "upb/message/internal/tagged_ptr.h" ] , "private-hdrs": ["upb/message/value.h"] , "deps": [ "base" , "base_internal" , "eps_copy_input_stream" , "hash" , "mem" , "message_types" , "mini_table" , "mini_table_internal" , "port" , "wire_reader" ] , "pure C": ["YES"] } , "message_compare": { "type": ["@", "rules", "CC", "library"] , "name": ["message_compare"] , "srcs": ["upb/message/compare.c"] , "hdrs": ["upb/message/compare.h"] , "deps": [ "base" , "message" , "message_internal" , "mini_table" , "mini_table_internal" , "port" ] , "pure C": ["YES"] } , "message_promote": { "type": ["@", "rules", "CC", "library"] , "name": ["message_promote"] , "srcs": ["upb/message/promote.c"] , "hdrs": ["upb/message/promote.h"] , "deps": [ "base" , "eps_copy_input_stream" , "mem" , "message" , "message_internal" , "mini_table" , "port" , "wire" , "wire_reader" ] , "pure C": ["YES"] } , "message_copy": { "type": ["@", "rules", "CC", "library"] , "name": ["message_copy"] , "srcs": ["upb/message/copy.c"] , "hdrs": ["upb/message/copy.h"] , "deps": [ "base" , "base_internal" , "mem" , "message" , "message_internal" , "mini_table" , "mini_table_internal" , "port" ] , "pure C": ["YES"] } , "message_split64": { "type": ["@", "rules", "CC", "library"] , "name": ["message_split64"] , "hdrs": ["upb/message/accessors_split64.h"] , "deps": ["message", "port"] , "pure C": ["YES"] } , "message_types": { "type": ["@", "rules", "CC", "library"] , "name": ["message_types"] , "hdrs": ["upb/message/internal/map_entry.h", "upb/message/internal/types.h"] , "deps": ["base", "hash", "port"] , "pure C": ["YES"] } , "message_value": { "type": ["@", "rules", "CC", "library"] , "name": ["message_value"] , "hdrs": ["upb/message/value.h"] , "deps": ["base", "message_tagged_ptr", "message_types"] , "pure C": ["YES"] } , "mini_descriptor": { "type": ["@", "rules", "CC", "library"] , "name": ["mini_descriptor"] , "srcs": [ "upb/mini_descriptor/build_enum.c" , "upb/mini_descriptor/decode.c" , "upb/mini_descriptor/link.c" ] , "hdrs": [ "upb/mini_descriptor/build_enum.h" , "upb/mini_descriptor/decode.h" , "upb/mini_descriptor/link.h" ] , "deps": [ "base" , "base_internal" , "mem" , "message_types" , "mini_descriptor_internal" , "mini_table" , "mini_table_internal" , "port" ] , "pure C": ["YES"] } , "mini_descriptor_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["mini_descriptor_internal"] , "srcs": [ "upb/mini_descriptor/internal/base92.c" , "upb/mini_descriptor/internal/encode.c" ] , "hdrs": [ "upb/mini_descriptor/internal/base92.h" , "upb/mini_descriptor/internal/decoder.h" , "upb/mini_descriptor/internal/encode.h" , "upb/mini_descriptor/internal/encode.hpp" , "upb/mini_descriptor/internal/modifiers.h" , "upb/mini_descriptor/internal/wire_constants.h" ] , "deps": ["base", "base_internal", "port"] , "pure C": ["YES"] } , "mini_table_compat": { "type": ["@", "rules", "CC", "library"] , "name": ["mini_table_compat"] , "srcs": ["upb/mini_table/compat.c"] , "hdrs": ["upb/mini_table/compat.h"] , "deps": ["base", "hash", "mem", "mini_table", "port"] , "pure C": ["YES"] } , "mini_table": { "type": ["@", "rules", "CC", "library"] , "name": ["mini_table"] , "srcs": ["upb/mini_table/extension_registry.c", "upb/mini_table/message.c"] , "hdrs": [ "upb/mini_table/enum.h" , "upb/mini_table/extension.h" , "upb/mini_table/extension_registry.h" , "upb/mini_table/field.h" , "upb/mini_table/file.h" , "upb/mini_table/message.h" , "upb/mini_table/sub.h" ] , "deps": ["base", "hash", "mem", "mini_table_internal", "port"] , "pure C": ["YES"] } , "mini_table_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["mini_table_internal"] , "srcs": ["upb/mini_table/internal/message.c"] , "hdrs": [ "upb/mini_table/internal/enum.h" , "upb/mini_table/internal/extension.h" , "upb/mini_table/internal/field.h" , "upb/mini_table/internal/file.h" , "upb/mini_table/internal/message.h" , "upb/mini_table/internal/size_log2.h" , "upb/mini_table/internal/sub.h" ] , "deps": ["base", "hash", "mem", "message_types", "port"] , "pure C": ["YES"] } , "port": { "type": ["@", "rules", "CC", "library"] , "name": ["port"] , "hdrs": [ "upb/port/atomic.h" , "upb/port/vsnprintf_compat.h" , "upb/port/def.inc" , "upb/port/undef.inc" ] , "pure C": ["YES"] } , "text": { "type": ["@", "rules", "CC", "library"] , "name": ["text"] , "srcs": ["upb/text/encode.c"] , "hdrs": ["upb/text/encode.h", "upb/text/options.h"] , "deps": [ "base" , "eps_copy_input_stream" , "lex" , "message" , "message_internal" , "message_types" , "port" , "reflection" , "text_internal" , "wire_reader" , ["@", "protobuf", "third_party/utf8_range", "utf8_range"] ] , "pure C": ["YES"] } , "text_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["text_internal"] , "srcs": ["upb/text/internal/encode.c"] , "hdrs": ["upb/text/internal/encode.h", "upb/text/options.h"] , "deps": [ "base" , "eps_copy_input_stream" , "lex" , "message" , "message_internal" , "port" , "wire_reader" , ["@", "protobuf", "third_party/utf8_range", "utf8_range"] ] , "pure C": ["YES"] } , "wire": { "type": ["@", "rules", "CC", "library"] , "name": ["wire"] , "srcs": [ "upb/wire/decode.c" , "upb/wire/encode.c" , "upb/wire/internal/decode_fast.c" ] , "hdrs": [ "upb/wire/decode.h" , "upb/wire/encode.h" , "upb/wire/internal/decode_fast.h" ] , "private-hdrs": ["upb/wire/internal/constants.h", "upb/wire/internal/decoder.h"] , "deps": [ "base" , "base_internal" , "eps_copy_input_stream" , "hash" , "mem" , "mem_internal" , "message" , "message_internal" , "message_types" , "mini_table" , "mini_table_internal" , "port" , "wire_reader" , ["@", "protobuf", "third_party/utf8_range", "utf8_range"] ] , "pure C": ["YES"] } , "wire_reader": { "type": ["@", "rules", "CC", "library"] , "name": ["wire_reader"] , "srcs": ["upb/wire/reader.c"] , "hdrs": ["upb/wire/internal/reader.h", "upb/wire/reader.h", "upb/wire/types.h"] , "deps": ["base_internal", "eps_copy_input_stream", "port"] , "pure C": ["YES"] } , "eps_copy_input_stream": { "type": ["@", "rules", "CC", "library"] , "name": ["eps_copy_input_stream"] , "srcs": ["upb/wire/eps_copy_input_stream.c"] , "hdrs": ["upb/wire/eps_copy_input_stream.h"] , "deps": ["mem", "port"] , "pure C": ["YES"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/utf8_range/000077500000000000000000000000001516554100600262775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/utf8_range/TARGETS.protobuf000066400000000000000000000021461516554100600311750ustar00rootroot00000000000000{ "utf8_validity": { "type": "export" , "target": "utf8_validity_internal" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "utf8_validity_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["utf8_validity"] , "hdrs": ["utf8_validity.h"] , "srcs": ["utf8_validity.cc"] , "deps": ["utf8_range", ["@", "absl", "absl/strings", "strings"]] } , "utf8_range": { "type": "export" , "target": "utf8_range_internal" , "flexible_config": [ "OS" , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "CC" , "CFLAGS" , "ADD_CFLAGS" , "PKG_CONFIG_ARGS" ] } , "utf8_range_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["utf8_range"] , "hdrs": ["utf8_range.h"] , "srcs": ["utf8_range.c"] , "pure C": ["YES"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/xxhash/000077500000000000000000000000001516554100600255405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/third_party/xxhash/TARGETS.grpc000066400000000000000000000002041516554100600275220ustar00rootroot00000000000000{ "xxhash": { "type": ["@", "rules", "CC", "library"] , "name": ["xxhash"] , "pure C": ["YES"] , "hdrs": ["xxhash.h"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/upb/000077500000000000000000000000001516554100600224725ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/upb/TARGETS.protobuf000066400000000000000000000003131516554100600253620ustar00rootroot00000000000000{ "port_hdrs": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "port/atomic.h" , "port/def.inc" , "port/undef.inc" , "port/vsnprintf_compat.h" ] , "stage": ["upb"] } } just-buildsystem-justbuild-b1fb5fa/etc/import/upb_generator/000077500000000000000000000000001516554100600245405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/import/upb_generator/TARGETS.protobuf000066400000000000000000000005611516554100600274350ustar00rootroot00000000000000{ "names_srcs": { "type": ["@", "rules", "data", "staged"] , "srcs": ["common/names.cc", "minitable/names.cc", "minitable/names_internal.cc"] , "stage": ["upb_generator"] } , "names_hdrs": { "type": ["@", "rules", "data", "staged"] , "srcs": ["common/names.h", "minitable/names.h", "minitable/names_internal.h"] , "stage": ["upb_generator"] } } just-buildsystem-justbuild-b1fb5fa/etc/patches/000077500000000000000000000000001516554100600220215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/patches/TARGETS000066400000000000000000000000031516554100600230460ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/etc/patches/options.h.diff000066400000000000000000000025711516554100600246010ustar00rootroot00000000000000--- options.h.orig 2023-08-21 14:27:30.600786502 +0200 +++ options.h 2023-08-21 14:28:34.951926481 +0200 @@ -94,7 +94,7 @@ // User code should not inspect this macro. To check in the preprocessor if // absl::any is a typedef of std::any, use the feature macro ABSL_USES_STD_ANY. -#define ABSL_OPTION_USE_STD_ANY 2 +#define ABSL_OPTION_USE_STD_ANY 0 // ABSL_OPTION_USE_STD_OPTIONAL @@ -121,7 +121,7 @@ // absl::optional is a typedef of std::optional, use the feature macro // ABSL_USES_STD_OPTIONAL. -#define ABSL_OPTION_USE_STD_OPTIONAL 2 +#define ABSL_OPTION_USE_STD_OPTIONAL 0 // ABSL_OPTION_USE_STD_STRING_VIEW @@ -148,7 +148,7 @@ // absl::string_view is a typedef of std::string_view, use the feature macro // ABSL_USES_STD_STRING_VIEW. -#define ABSL_OPTION_USE_STD_STRING_VIEW 2 +#define ABSL_OPTION_USE_STD_STRING_VIEW 0 // ABSL_OPTION_USE_STD_VARIANT // @@ -174,7 +174,7 @@ // absl::variant is a typedef of std::variant, use the feature macro // ABSL_USES_STD_VARIANT. -#define ABSL_OPTION_USE_STD_VARIANT 2 +#define ABSL_OPTION_USE_STD_VARIANT 0 // ABSL_OPTION_USE_STD_ORDERING // @@ -201,7 +201,7 @@ // the ordering types are aliases of std:: ordering types, use the feature macro // ABSL_USES_STD_ORDERING. -#define ABSL_OPTION_USE_STD_ORDERING 2 +#define ABSL_OPTION_USE_STD_ORDERING 0 // ABSL_OPTION_USE_INLINE_NAMESPACE // ABSL_OPTION_INLINE_NAMESPACE_NAME just-buildsystem-justbuild-b1fb5fa/etc/repos.in.json000066400000000000000000000513761516554100600230360ustar00rootroot00000000000000{ "main": "just" , "keep": ["just tests", "lint"] , "repositories": { "just": { "repository": {"type": "file", "path": "."} , "bindings": { "rules": "rules-just" , "gsl": "com_github_microsoft_gsl" , "cli11": "cli11" , "json": "json" , "fmt": "fmt" , "ssl": "ssl" , "grpc": "com_github_grpc_grpc" , "googleapis": "google_apis" , "bazel_remote_apis": "bazel_remote_apis" , "libgit2": "com_github_libgit2_libgit2" , "protoc": "protobuf" , "libcurl": "com_github_curl_curl" , "libarchive": "com_github_libarchive_libarchive" } , "bootstrap": {"link": ["-pthread"]} , "bootstrap_local": {"link": ["-pthread"]} } , "just tests": { "repository": {"type": "file", "path": "test"} , "bindings": { "src": "just" , "rules": "rules-just" , "just-distfiles": "just-distfiles" , "catch2": "catch2" , "gsl": "com_github_microsoft_gsl" , "cli11": "cli11" , "json": "json" , "fmt": "fmt" , "ssl": "ssl" , "grpc": "com_github_grpc_grpc" , "googleapis": "google_apis" , "bazel_remote_apis": "bazel_remote_apis" , "libgit2": "com_github_libgit2_libgit2" , "protoc": "protobuf" } } , "lint": { "repository": {"type": "file", "path": "lint"} , "bindings": { "rules": "rules-just" , "src": "just" , "tests": "just tests" , "clang": "clang-18" } } , "just-distfiles": { "repository": { "type": "distdir" , "repositories": [ "com_github_microsoft_gsl" , "cli11" , "json" , "fmt" , "ssl" , "protobuf" , "bazel_remote_apis" , "google_apis" , "com_google_absl" , "zlib" , "re2" , "com_github_cares_cares" , "com_github_grpc_grpc" , "com_github_libgit2_libgit2" , "com_github_curl_curl" , "com_github_libarchive_libarchive" , "lzma" , "bzip2" ] } , "target_root": "import targets" , "target_file_name": "TARGETS.distfiles" } , "toolchain": { "repository": {"type": "file", "path": "etc/toolchain", "pragma": {"to_git": true}} , "rule_root": "rules" } , "defaults": { "repository": {"type": "file", "path": "etc/defaults", "pragma": {"to_git": true}} } , "patches": { "repository": {"type": "file", "path": "etc/patches", "pragma": {"to_git": true}} } , "rules": { "repository": {"type": "file", "path": "rules", "pragma": {"to_git": true}} , "target_root": "defaults" , "rule_root": "rules" , "bindings": {"toolchain": "toolchain"} } , "rules-just": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.just" , "bindings": {"base": "rules", "protoc": "protobuf", "grpc": "com_github_grpc_grpc"} } , "rules-boringssl": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.boringssl" , "bindings": {"base": "rules"} } , "rules-protobuf": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.protobuf" , "bindings": {"base": "rules"} } , "rules-grpc": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.grpc" , "bindings": {"base": "rules", "protoc": "protobuf"} } , "rules-absl": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.absl" , "bindings": {"base": "rules"} } , "rules-re2": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.re2" , "bindings": {"base": "rules"} } , "rules-git2": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.git2" , "bindings": {"base": "rules"} } , "rules-curl": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.curl" , "bindings": {"base": "rules"} } , "rules-bzip2": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.bzip2" , "bindings": {"base": "rules"} } , "rules-lzma": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.lzma" , "bindings": {"base": "rules"} } , "rules-archive": { "repository": "rules" , "target_root": "defaults" , "rule_root": "rules" , "target_file_name": "TARGETS.archive" , "bindings": {"base": "rules"} } , "import targets": { "repository": {"type": "file", "path": "etc/import", "pragma": {"to_git": true}} , "pkg_bootstrap": {"local_path": "etc/import.pkgconfig"} } , "com_github_microsoft_gsl": { "repository": { "type": "archive" , "content": "386f0a89a47763481223a63d5026215af8d3c827" , "fetch": "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" , "sha256": "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" , "sha512": "7fa7446796c6bf82fb3bff09f86a69c446a27be528bef3b17c8bc5ad2f24d5cf86bdb3d3813ecb44726e8f395020180e97e41027330d1fbf545cc0f0b44aac29" , "subdir": "GSL-4.0.0/include/gsl" } , "target_root": "import targets" , "target_file_name": "TARGETS.gsl" , "bindings": {"rules": "rules"} , "bootstrap": {"include_name": "gsl"} , "pkg_bootstrap": {"local_path": "include/gsl"} } , "cli11": { "repository": { "type": "archive" , "content": "6689a311c4e3f31e0428a6135ed03f2ccac448f8" , "fetch": "https://github.com/CLIUtils/CLI11/archive/refs/tags/v2.5.0.tar.gz" , "sha256": "17e02b4cddc2fa348e5dbdbb582c59a3486fa2b2433e70a0c3bacb871334fd55" , "sha512": "895fb61e4c9974ee8e8d4681fb880a10126a412f24bb147d558d465d78fe784a044c5443edf1ce20fc9936901073073d795b034e0c02bdb3c8aa74c9d6ac811c" , "subdir": "CLI11-2.5.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.cli11" , "bindings": {"rules": "rules"} , "bootstrap": {"include_dir": "include/CLI", "include_name": "CLI"} } , "json": { "repository": { "type": "zip" , "content": "733571f96614cc01fcebf76c8359f52706677c61" , "fetch": "https://github.com/nlohmann/json/releases/download/v3.11.3/include.zip" , "sha256": "a22461d13119ac5c78f205d3df1db13403e58ce1bb1794edc9313677313f4a9d" , "sha512": "8d923e2586acf736fc1886bf1839ca0126444ec60ce93a4bd18c21eef4475dff6f608203e42bf4968878dc50727a8c20c517dd8c1ac5c6b0bb6a95f2dce5546e" , "subdir": "include/nlohmann" } , "target_root": "import targets" , "target_file_name": "TARGETS.json" , "bindings": {"rules": "rules"} , "bootstrap": {"include_name": "nlohmann"} } , "fmt": { "repository": { "type": "zip" , "content": "da6ad435963d4578c63c723e61a1e6b136fd61d8" , "fetch": "https://github.com/fmtlib/fmt/releases/download/11.2.0/fmt-11.2.0.zip" , "sha256": "203eb4e8aa0d746c62d8f903df58e0419e3751591bb53ff971096eaa0ebd4ec3" , "sha512": "75586d02284a33c0c101b6e78cbb1d61f169610ae9027ddfc20936751a8c2ac4453f3046e7b05fa167a8f8eedeafde0f4cb0bff4f798c17c80994521f660174d" , "subdir": "fmt-11.2.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.fmt" , "bindings": {"rules": "rules"} , "bootstrap": { "include_dir": "include/fmt" , "build": "cd src && {cxx} {cxxflags} -I ../include -c os.cc format.cc && {ar} cqs ../libfmt.a *.o" , "link": ["-lfmt"] } , "pkg_bootstrap": {"link": ["-lfmt"], "link_dirs": ["lib"]} } , "ssl": { "repository": { "type": "archive" , "content": "d9ff3aa3d22337e93cb0a0e2df00ca8abea2fac6" , "fetch": "https://github.com/google/boringssl/archive/dec0d8f681348af8bb675e07bd89989665fca8bc.tar.gz" , "sha256": "2f12c33d2cf25a658a1b981fb96923dac87e9175fb20e45db6950ee36c526356" , "sha512": "51f91bee640e38ce20c180195de6811b5be7522240faae2b57158fa9c298ed09292ecb09c770df5d4fc6a3100bc2459de4e9d312e315265c1ea7ade347bad3f2" , "subdir": "boringssl-dec0d8f681348af8bb675e07bd89989665fca8bc" , "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/dec0d8f681348af8bb675e07bd89989665fca8bc.tar.gz" ] } , "target_root": "import targets" , "target_file_name": "TARGETS.boringssl" , "bindings": {"rules": "rules-boringssl", "patches": "patches"} , "bootstrap": { "arch_map": {"arm64": "aarch64"} , "build": "{cxx} {cxxflags} -I . -I src/include -c `find . '(' -ipath './src/crypto/*.cc' -o -ipath './src/gen/crypto/*.cc' -o -ipath './src/crypto/*.S' -o -ipath './src/gen/bcm/*.S' -o -ipath './src/gen/crypto/*.S' -o -ipath './src/third_party/fiat/asm/*.S' ')' -type f ! -ipath '*_test.*' ! -ipath '*/test/*'` && {ar} cqs libcrypto.a *.o" , "link": ["-lcrypto", "-pthread"] , "include_dir": "src/include/openssl" , "include_name": "openssl" } , "pkg_bootstrap": {"link": ["-lcrypto", "-pthread"], "link_dirs": ["lib"]} } , "protobuf": { "repository": { "type": "archive" , "content": "e8787a2f6d661aaaec426743c4e9667be7c8ba2f" , "fetch": "https://github.com/protocolbuffers/protobuf/releases/download/v29.0/protobuf-29.0.tar.gz" , "sha256": "10a0d58f39a1a909e95e00e8ba0b5b1dc64d02997f741151953a2b3659f6e78c" , "sha512": "c5637486a533557ea909d1f880b0f0064fff0c4665612e023170941310c45bf8e7373d2c67de621824b056530e98792c00799d71ec4ff7b6af9142cdc4cb8dee" , "subdir": "protobuf-29.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.protobuf" , "bindings": {"rules": "rules-protobuf", "zlib": "zlib", "absl": "com_google_absl"} , "pkg_bootstrap": { "copy": ["bin/protoc", "include/google/protobuf", "proto/google/protobuf"] } } , "bazel_remote_apis": { "repository": { "type": "archive" , "content": "b2c8d2624519413a67d354d8e2b1b707e29482b1" , "fetch": "https://github.com/bazelbuild/remote-apis/archive/9ef19c6b5fbf77d6dd9d84d75fbb5a20a6b62ef1.tar.gz" , "sha256": "ccf57539b6347ceb0aa7e93ee43b9fff1aeb496d36cc097da918c4a35ef65839" , "sha512": "6fc84bc0b3f0730651ec9f7fe03e343c02acaabfef80cd3057343b6c6b935820e4fea27e0c8f0f0d86ccd9c3cbf82461e75b66740326113cf647339007fb3c0c" , "subdir": "remote-apis-9ef19c6b5fbf77d6dd9d84d75fbb5a20a6b62ef1" } , "target_root": "import targets" , "target_file_name": "TARGETS.bazel_remote_apis" , "bindings": {"rules": "rules", "google_apis": "google_apis"} , "pkg_bootstrap": {"local_path": "include"} } , "google_apis": { "repository": { "type": "archive" , "content": "db3c51a8fd9c923a4e4908d8bcd7dd4642cc4664" , "fetch": "https://github.com/googleapis/googleapis/archive/fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0.tar.gz" , "sha256": "0513f0f40af63bd05dc789cacc334ab6cec27cc89db596557cb2dfe8919463e4" , "sha512": "d77ea83f8e68e3c0b667e7de43c2cd28b0ca7b969b2cf127b3873fc19f330ad85afb314bef4174a4e11ed68b620e43853d8b44eb833c5eca7e820ca21c1c3e15" , "subdir": "googleapis-fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0" , "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0.tar.gz" ] } , "target_root": "import targets" , "target_file_name": "TARGETS.google_apis" , "bindings": {"rules": "rules", "patches": "patches"} , "pkg_bootstrap": {"local_path": "include"} } , "com_google_absl": { "repository": { "type": "archive" , "content": "b5cc395c49afa2a2dec1d9c4072500494b126974" , "fetch": "https://github.com/abseil/abseil-cpp/releases/download/20240722.0/abseil-cpp-20240722.0.tar.gz" , "sha256": "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3" , "sha512": "bd2cca8f007f2eee66f51c95a979371622b850ceb2ce3608d00ba826f7c494a1da0fba3c1427728f2c173fe50d59b701da35c2c9fdad2752a5a49746b1c8ef31" , "subdir": "abseil-cpp-20240722.0" , "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz" ] } , "target_root": "import targets" , "target_file_name": "TARGETS.absl" , "bindings": {"rules": "rules-absl", "patches": "patches"} } , "zlib": { "repository": { "type": "archive" , "content": "53fa48bf97f0ee0f42c62743b018507a6583ec3e" , "fetch": "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz" , "sha256": "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" , "sha512": "580677aad97093829090d4b605ac81c50327e74a6c2de0b85dd2e8525553f3ddde17556ea46f8f007f89e435493c9a20bc997d1ef1c1c2c23274528e3c46b94f" , "subdir": "zlib-1.3.1" } , "target_root": "import targets" , "target_file_name": "TARGETS.zlib" , "bindings": {"rules": "rules"} } , "bzip2": { "repository": { "type": "archive" , "content": "f9d91f2012aedeedcfd3bd918124ca11f0160373" , "fetch": "https://github.com/libarchive/bzip2/archive/refs/tags/bzip2-1.0.8.tar.gz" , "sha256": "db106b740252669664fd8f3a1c69fe7f689d5cd4b132f82ba82b9afba27627df" , "sha512": "596d1b304f1f2d64b020d04845db10a2330c7f614a9fd0b5344afff65877d2141b3fcaa43d9e2dbc2f6a7929a1dab07df54d3d4bd69678b53906472958c7b80c" , "subdir": "bzip2-bzip2-1.0.8" } , "target_root": "import targets" , "target_file_name": "TARGETS.bzip2" , "bindings": {"rules": "rules-bzip2"} } , "lzma": { "repository": { "type": "archive" , "content": "80e67abd2e08a54ec21f195b3e9e4abfc64ba7e1" , "fetch": "https://github.com/tukaani-project/xz/releases/download/v5.6.3/xz-5.6.3.tar.gz" , "sha256": "b1d45295d3f71f25a4c9101bd7c8d16cb56348bbef3bbc738da0351e17c73317" , "sha512": "b07b45e18615d1946e9d12157af99b54700d757832a638fccab70549574dcd7f28e69e71cc4c0b9c808959f818e79b668a5ccf108429ea0f40b6125bfd55d274" , "subdir": "xz-5.6.3" } , "target_root": "import targets" , "target_file_name": "TARGETS.lzma" , "bindings": {"rules": "rules-lzma"} } , "re2": { "repository": { "type": "archive" , "content": "8eebd70d7ebd33ac5d736090cecd7cfe1831b9e3" , "fetch": "https://github.com/google/re2/archive/refs/tags/2022-04-01.tar.gz" , "sha256": "1ae8ccfdb1066a731bba6ee0881baad5efd2cd661acd9569b689f2586e1a50e9" , "sha512": "fc3d7cc1ee6bd771719845566d83ffc8c4e19d838748e842a1e19c7564473c9a0a061bebb3966ffa82de6515346f9bbddc2d94ceb3de89233f58826774bd7ce7" , "subdir": "re2-2022-04-01" , "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/re2/archive/2022-04-01.tar.gz" ] } , "target_root": "import targets" , "target_file_name": "TARGETS.re2" , "bindings": {"rules": "rules-re2"} } , "com_github_cares_cares": { "repository": { "type": "archive" , "content": "4237a53f7a5dc1cfb4d04a6b6374f8674c32e271" , "fetch": "https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz" , "sha256": "321700399b72ed0e037d0074c629e7741f6b2ec2dda92956abe3e9671d3e268e" , "sha512": "466a94efda626e815a6ef7a890637056339f883d549ea6055e289fd8cd2391130e5682c905c0fb3bd7e955af7f6deb793562c170eb0ee066a4a62085a82ba470" , "subdir": "c-ares-1.19.1" } , "target_root": "import targets" , "target_file_name": "TARGETS.cares" , "bindings": {"rules": "rules", "grpc": "com_github_grpc_grpc"} } , "com_github_grpc_grpc": { "repository": { "type": "archive" , "content": "87c37bc5be5d6ded368667d02958d488b94d6143" , "fetch": "https://github.com/grpc/grpc/archive/refs/tags/v1.70.2.tar.gz" , "sha256": "92f240f7267ed6cd8ba2be4c59a3b5b6ec0c4b4c466071b1e1d62165b25acf64" , "sha512": "4afd34b36b4d7c41260abad6d921154a3e7176a7a46de6ae97335697db5d11f61e37b08889c007463fc8f3e800788585f5f7d5b2cec12f89b69fc9bc15efd17f" , "subdir": "grpc-1.70.2" , "pragma": {"special": "ignore"} } , "target_root": "import targets" , "target_file_name": "TARGETS.grpc" , "rule_file_name": "RULES.grpc" , "bindings": { "rules": "rules-grpc" , "protobuf": "protobuf" , "ssl": "ssl" , "absl": "com_google_absl" , "zlib": "zlib" , "re2": "re2" , "cares": "com_github_cares_cares" , "google_apis": "google_apis" } , "pkg_bootstrap": { "copy": [ "bin/grpc_cpp_plugin" , "include/grpc" , "include/grpcpp" , "include/grpc++" ] } } , "com_github_libgit2_libgit2": { "repository": { "type": "archive" , "content": "80f999c62e8da8da7d64fe543c8eeb9953bf6974" , "fetch": "https://github.com/libgit2/libgit2/archive/refs/tags/v1.7.2.tar.gz" , "sha256": "de384e29d7efc9330c6cdb126ebf88342b5025d920dcb7c645defad85195ea7f" , "sha512": "825737e4a1991fba50ea535f15b0e560ebe76ead752e04aeba36925b944d0da77fe9826a70980a1aa3d0bf9afbedfab79dd92e799c9252931384c89ebec9b012" , "subdir": "libgit2-1.7.2" } , "target_root": "import targets" , "target_file_name": "TARGETS.git2" , "bindings": {"rules": "rules-git2", "zlib": "zlib", "ssl": "ssl"} , "bootstrap": {"include_dir": "include", "include_name": "."} , "pkg_bootstrap": {"copy": ["git2.h", "git2"]} } , "catch2": { "repository": { "type": "archive" , "content": "5af653af0c54ad580176a8cadb5a1cef8c880372" , "fetch": "https://github.com/catchorg/Catch2/archive/refs/tags/v3.5.3.tar.gz" , "sha256": "8d723b0535c94860ef8cf6231580fa47d67a3416757ecb10639e40d748ab6c71" , "sha512": "57c996f679cbad212cb0fde39e506bade37bd559c0e93e20f407f2a2f029e98b78661e10257f9c8e4cb5fd7d52d0ea1eae3d4a1f989c6d66fcb281e32e1688f6" , "subdir": "Catch2-3.5.3" } , "target_root": "import targets" , "target_file_name": "TARGETS.catch2" , "bindings": {"rules": "rules"} , "bootstrap": {"drop": true} } , "com_github_curl_curl": { "repository": { "type": "archive" , "content": "7b7ace4d7cc7d4059163bbd789ef7071e5d326e5" , "fetch": "https://github.com/curl/curl/releases/download/curl-8_6_0/curl-8.6.0.tar.gz" , "sha256": "9c6db808160015f30f3c656c0dec125feb9dc00753596bf858a272b5dd8dc398" , "sha512": "43fdb6b81b394f3382d353d5f57673b2b3d26cfe34b25d08a526bc0597f508d5298e5a7088d0736d1f139cad19cd922affa51533c3a5a4bb5f2de68891c2958d" , "subdir": "curl-8.6.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.curl" , "bindings": { "rules": "rules-curl" , "zlib": "zlib" , "ssl": "ssl" , "cares": "com_github_cares_cares" } } , "com_github_libarchive_libarchive": { "repository": { "type": "archive" , "content": "994435922d1ce63b52f6420f90b1b2a9f6670c39" , "fetch": "https://github.com/libarchive/libarchive/releases/download/v3.7.7/libarchive-3.7.7.tar.gz" , "sha256": "4cc540a3e9a1eebdefa1045d2e4184831100667e6d7d5b315bb1cbc951f8ddff" , "sha512": "cce6eecfcd33d228bd1b1162a90bad63750adb53ac4edcaed34e2fdc30b6ba211cf1fd25d4b8761373949ceec266478b09bd70ffa4e374803a29e8573d6d149e" , "subdir": "libarchive-3.7.7" } , "target_root": "import targets" , "target_file_name": "TARGETS.archive" , "bindings": { "rules": "rules-archive" , "zlib": "zlib" , "ssl": "ssl" , "bzip2": "bzip2" , "lzma": "lzma" } } , "repo": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} , "format-json infra": { "repository": {"type": "file", "path": "format-json", "pragma": {"to_git": true}} } , "repo tree": { "repository": {"type": "tree structure", "repo": "repo"} , "target_file_name": "TARGETS.tasks" , "target_root": "format-json infra" , "bindings": {"format": "format-json infra"} } , "format-json/tasks": { "repository": {"type": "computed", "repo": "repo tree", "target": ["", ""]} } , "format-json/bin": { "repository": {"type": "file", "path": "bin", "pragma": {"to_git": true}} , "target_root": "format-json infra" } , "format-json": { "repository": {"type": "file", "path": "."} , "target_root": "format-json/tasks" , "rule_root": "format-json infra" , "bindings": {"bin": "format-json/bin"} } } , "imports": [ { "source": "git" , "url": "https://github.com/just-buildsystem/bootstrappable-toolchain" , "branch": "master" , "repos": [{"repo": "clang-18.1.8-native", "alias": "clang-18"}] } ] } just-buildsystem-justbuild-b1fb5fa/etc/repos.json000066400000000000000000001272751516554100600224330ustar00rootroot00000000000000{ "main": "just", "repositories": { "just": { "repository": { "type": "file", "path": "." }, "bindings": { "rules": "rules-just", "gsl": "com_github_microsoft_gsl", "cli11": "cli11", "json": "json", "fmt": "fmt", "ssl": "ssl", "grpc": "com_github_grpc_grpc", "googleapis": "google_apis", "bazel_remote_apis": "bazel_remote_apis", "libgit2": "com_github_libgit2_libgit2", "protoc": "protobuf", "libcurl": "com_github_curl_curl", "libarchive": "com_github_libarchive_libarchive" }, "bootstrap": { "link": [ "-pthread" ] }, "bootstrap_local": { "link": [ "-pthread" ] } }, "just tests": { "repository": { "type": "file", "path": "test" }, "bindings": { "src": "just", "rules": "rules-just", "just-distfiles": "just-distfiles", "catch2": "catch2", "gsl": "com_github_microsoft_gsl", "cli11": "cli11", "json": "json", "fmt": "fmt", "ssl": "ssl", "grpc": "com_github_grpc_grpc", "googleapis": "google_apis", "bazel_remote_apis": "bazel_remote_apis", "libgit2": "com_github_libgit2_libgit2", "protoc": "protobuf" } }, "lint": { "repository": { "type": "file", "path": "lint" }, "bindings": { "rules": "rules-just", "src": "just", "tests": "just tests", "clang": "clang-18" } }, "just-distfiles": { "repository": { "type": "distdir", "repositories": [ "com_github_microsoft_gsl", "cli11", "json", "fmt", "ssl", "protobuf", "bazel_remote_apis", "google_apis", "com_google_absl", "zlib", "re2", "com_github_cares_cares", "com_github_grpc_grpc", "com_github_libgit2_libgit2", "com_github_curl_curl", "com_github_libarchive_libarchive", "lzma", "bzip2" ] }, "target_root": "import targets", "target_file_name": "TARGETS.distfiles" }, "toolchain": { "repository": { "type": "file", "path": "etc/toolchain", "pragma": { "to_git": true } }, "rule_root": "rules" }, "defaults": { "repository": { "type": "file", "path": "etc/defaults", "pragma": { "to_git": true } } }, "patches": { "repository": { "type": "file", "path": "etc/patches", "pragma": { "to_git": true } } }, "rules": { "repository": { "type": "file", "path": "rules", "pragma": { "to_git": true } }, "target_root": "defaults", "rule_root": "rules", "bindings": { "toolchain": "toolchain" } }, "rules-just": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.just", "bindings": { "base": "rules", "protoc": "protobuf", "grpc": "com_github_grpc_grpc" } }, "rules-boringssl": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.boringssl", "bindings": { "base": "rules" } }, "rules-protobuf": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.protobuf", "bindings": { "base": "rules" } }, "rules-grpc": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.grpc", "bindings": { "base": "rules", "protoc": "protobuf" } }, "rules-absl": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.absl", "bindings": { "base": "rules" } }, "rules-re2": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.re2", "bindings": { "base": "rules" } }, "rules-git2": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.git2", "bindings": { "base": "rules" } }, "rules-curl": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.curl", "bindings": { "base": "rules" } }, "rules-bzip2": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.bzip2", "bindings": { "base": "rules" } }, "rules-lzma": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.lzma", "bindings": { "base": "rules" } }, "rules-archive": { "repository": "rules", "target_root": "defaults", "rule_root": "rules", "target_file_name": "TARGETS.archive", "bindings": { "base": "rules" } }, "import targets": { "repository": { "type": "file", "path": "etc/import", "pragma": { "to_git": true } }, "pkg_bootstrap": { "local_path": "etc/import.pkgconfig" } }, "com_github_microsoft_gsl": { "repository": { "type": "archive", "content": "386f0a89a47763481223a63d5026215af8d3c827", "fetch": "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz", "sha256": "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9", "sha512": "7fa7446796c6bf82fb3bff09f86a69c446a27be528bef3b17c8bc5ad2f24d5cf86bdb3d3813ecb44726e8f395020180e97e41027330d1fbf545cc0f0b44aac29", "subdir": "GSL-4.0.0/include/gsl" }, "target_root": "import targets", "target_file_name": "TARGETS.gsl", "bindings": { "rules": "rules" }, "bootstrap": { "include_name": "gsl" }, "pkg_bootstrap": { "local_path": "include/gsl" } }, "cli11": { "repository": { "type": "archive", "content": "6689a311c4e3f31e0428a6135ed03f2ccac448f8", "fetch": "https://github.com/CLIUtils/CLI11/archive/refs/tags/v2.5.0.tar.gz", "sha256": "17e02b4cddc2fa348e5dbdbb582c59a3486fa2b2433e70a0c3bacb871334fd55", "sha512": "895fb61e4c9974ee8e8d4681fb880a10126a412f24bb147d558d465d78fe784a044c5443edf1ce20fc9936901073073d795b034e0c02bdb3c8aa74c9d6ac811c", "subdir": "CLI11-2.5.0" }, "target_root": "import targets", "target_file_name": "TARGETS.cli11", "bindings": { "rules": "rules" }, "bootstrap": { "include_dir": "include/CLI", "include_name": "CLI" } }, "json": { "repository": { "type": "zip", "content": "733571f96614cc01fcebf76c8359f52706677c61", "fetch": "https://github.com/nlohmann/json/releases/download/v3.11.3/include.zip", "sha256": "a22461d13119ac5c78f205d3df1db13403e58ce1bb1794edc9313677313f4a9d", "sha512": "8d923e2586acf736fc1886bf1839ca0126444ec60ce93a4bd18c21eef4475dff6f608203e42bf4968878dc50727a8c20c517dd8c1ac5c6b0bb6a95f2dce5546e", "subdir": "include/nlohmann" }, "target_root": "import targets", "target_file_name": "TARGETS.json", "bindings": { "rules": "rules" }, "bootstrap": { "include_name": "nlohmann" } }, "fmt": { "repository": { "type": "zip", "content": "da6ad435963d4578c63c723e61a1e6b136fd61d8", "fetch": "https://github.com/fmtlib/fmt/releases/download/11.2.0/fmt-11.2.0.zip", "sha256": "203eb4e8aa0d746c62d8f903df58e0419e3751591bb53ff971096eaa0ebd4ec3", "sha512": "75586d02284a33c0c101b6e78cbb1d61f169610ae9027ddfc20936751a8c2ac4453f3046e7b05fa167a8f8eedeafde0f4cb0bff4f798c17c80994521f660174d", "subdir": "fmt-11.2.0" }, "target_root": "import targets", "target_file_name": "TARGETS.fmt", "bindings": { "rules": "rules" }, "bootstrap": { "include_dir": "include/fmt", "build": "cd src && {cxx} {cxxflags} -I ../include -c os.cc format.cc && {ar} cqs ../libfmt.a *.o", "link": [ "-lfmt" ] }, "pkg_bootstrap": { "link": [ "-lfmt" ], "link_dirs": [ "lib" ] } }, "ssl": { "repository": { "type": "archive", "content": "d9ff3aa3d22337e93cb0a0e2df00ca8abea2fac6", "fetch": "https://github.com/google/boringssl/archive/dec0d8f681348af8bb675e07bd89989665fca8bc.tar.gz", "sha256": "2f12c33d2cf25a658a1b981fb96923dac87e9175fb20e45db6950ee36c526356", "sha512": "51f91bee640e38ce20c180195de6811b5be7522240faae2b57158fa9c298ed09292ecb09c770df5d4fc6a3100bc2459de4e9d312e315265c1ea7ade347bad3f2", "subdir": "boringssl-dec0d8f681348af8bb675e07bd89989665fca8bc", "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/dec0d8f681348af8bb675e07bd89989665fca8bc.tar.gz" ] }, "target_root": "import targets", "target_file_name": "TARGETS.boringssl", "bindings": { "rules": "rules-boringssl", "patches": "patches" }, "bootstrap": { "arch_map": { "arm64": "aarch64" }, "build": "{cxx} {cxxflags} -I . -I src/include -c `find . '(' -ipath './src/crypto/*.cc' -o -ipath './src/gen/crypto/*.cc' -o -ipath './src/crypto/*.S' -o -ipath './src/gen/bcm/*.S' -o -ipath './src/gen/crypto/*.S' -o -ipath './src/third_party/fiat/asm/*.S' ')' -type f ! -ipath '*_test.*' ! -ipath '*/test/*'` && {ar} cqs libcrypto.a *.o", "link": [ "-lcrypto", "-pthread" ], "include_dir": "src/include/openssl", "include_name": "openssl" }, "pkg_bootstrap": { "link": [ "-lcrypto", "-pthread" ], "link_dirs": [ "lib" ] } }, "protobuf": { "repository": { "type": "archive", "content": "e8787a2f6d661aaaec426743c4e9667be7c8ba2f", "fetch": "https://github.com/protocolbuffers/protobuf/releases/download/v29.0/protobuf-29.0.tar.gz", "sha256": "10a0d58f39a1a909e95e00e8ba0b5b1dc64d02997f741151953a2b3659f6e78c", "sha512": "c5637486a533557ea909d1f880b0f0064fff0c4665612e023170941310c45bf8e7373d2c67de621824b056530e98792c00799d71ec4ff7b6af9142cdc4cb8dee", "subdir": "protobuf-29.0" }, "target_root": "import targets", "target_file_name": "TARGETS.protobuf", "bindings": { "rules": "rules-protobuf", "zlib": "zlib", "absl": "com_google_absl" }, "pkg_bootstrap": { "copy": [ "bin/protoc", "include/google/protobuf", "proto/google/protobuf" ] } }, "bazel_remote_apis": { "repository": { "type": "archive", "content": "b2c8d2624519413a67d354d8e2b1b707e29482b1", "fetch": "https://github.com/bazelbuild/remote-apis/archive/9ef19c6b5fbf77d6dd9d84d75fbb5a20a6b62ef1.tar.gz", "sha256": "ccf57539b6347ceb0aa7e93ee43b9fff1aeb496d36cc097da918c4a35ef65839", "sha512": "6fc84bc0b3f0730651ec9f7fe03e343c02acaabfef80cd3057343b6c6b935820e4fea27e0c8f0f0d86ccd9c3cbf82461e75b66740326113cf647339007fb3c0c", "subdir": "remote-apis-9ef19c6b5fbf77d6dd9d84d75fbb5a20a6b62ef1" }, "target_root": "import targets", "target_file_name": "TARGETS.bazel_remote_apis", "bindings": { "rules": "rules", "google_apis": "google_apis" }, "pkg_bootstrap": { "local_path": "include" } }, "google_apis": { "repository": { "type": "archive", "content": "db3c51a8fd9c923a4e4908d8bcd7dd4642cc4664", "fetch": "https://github.com/googleapis/googleapis/archive/fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0.tar.gz", "sha256": "0513f0f40af63bd05dc789cacc334ab6cec27cc89db596557cb2dfe8919463e4", "sha512": "d77ea83f8e68e3c0b667e7de43c2cd28b0ca7b969b2cf127b3873fc19f330ad85afb314bef4174a4e11ed68b620e43853d8b44eb833c5eca7e820ca21c1c3e15", "subdir": "googleapis-fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0", "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/fe8ba054ad4f7eca946c2d14a63c3f07c0b586a0.tar.gz" ] }, "target_root": "import targets", "target_file_name": "TARGETS.google_apis", "bindings": { "rules": "rules", "patches": "patches" }, "pkg_bootstrap": { "local_path": "include" } }, "com_google_absl": { "repository": { "type": "archive", "content": "b5cc395c49afa2a2dec1d9c4072500494b126974", "fetch": "https://github.com/abseil/abseil-cpp/releases/download/20240722.0/abseil-cpp-20240722.0.tar.gz", "sha256": "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3", "sha512": "bd2cca8f007f2eee66f51c95a979371622b850ceb2ce3608d00ba826f7c494a1da0fba3c1427728f2c173fe50d59b701da35c2c9fdad2752a5a49746b1c8ef31", "subdir": "abseil-cpp-20240722.0", "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz" ] }, "target_root": "import targets", "target_file_name": "TARGETS.absl", "bindings": { "rules": "rules-absl", "patches": "patches" } }, "zlib": { "repository": { "type": "archive", "content": "53fa48bf97f0ee0f42c62743b018507a6583ec3e", "fetch": "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz", "sha256": "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23", "sha512": "580677aad97093829090d4b605ac81c50327e74a6c2de0b85dd2e8525553f3ddde17556ea46f8f007f89e435493c9a20bc997d1ef1c1c2c23274528e3c46b94f", "subdir": "zlib-1.3.1" }, "target_root": "import targets", "target_file_name": "TARGETS.zlib", "bindings": { "rules": "rules" } }, "bzip2": { "repository": { "type": "archive", "content": "f9d91f2012aedeedcfd3bd918124ca11f0160373", "fetch": "https://github.com/libarchive/bzip2/archive/refs/tags/bzip2-1.0.8.tar.gz", "sha256": "db106b740252669664fd8f3a1c69fe7f689d5cd4b132f82ba82b9afba27627df", "sha512": "596d1b304f1f2d64b020d04845db10a2330c7f614a9fd0b5344afff65877d2141b3fcaa43d9e2dbc2f6a7929a1dab07df54d3d4bd69678b53906472958c7b80c", "subdir": "bzip2-bzip2-1.0.8" }, "target_root": "import targets", "target_file_name": "TARGETS.bzip2", "bindings": { "rules": "rules-bzip2" } }, "lzma": { "repository": { "type": "archive", "content": "80e67abd2e08a54ec21f195b3e9e4abfc64ba7e1", "fetch": "https://github.com/tukaani-project/xz/releases/download/v5.6.3/xz-5.6.3.tar.gz", "sha256": "b1d45295d3f71f25a4c9101bd7c8d16cb56348bbef3bbc738da0351e17c73317", "sha512": "b07b45e18615d1946e9d12157af99b54700d757832a638fccab70549574dcd7f28e69e71cc4c0b9c808959f818e79b668a5ccf108429ea0f40b6125bfd55d274", "subdir": "xz-5.6.3" }, "target_root": "import targets", "target_file_name": "TARGETS.lzma", "bindings": { "rules": "rules-lzma" } }, "re2": { "repository": { "type": "archive", "content": "8eebd70d7ebd33ac5d736090cecd7cfe1831b9e3", "fetch": "https://github.com/google/re2/archive/refs/tags/2022-04-01.tar.gz", "sha256": "1ae8ccfdb1066a731bba6ee0881baad5efd2cd661acd9569b689f2586e1a50e9", "sha512": "fc3d7cc1ee6bd771719845566d83ffc8c4e19d838748e842a1e19c7564473c9a0a061bebb3966ffa82de6515346f9bbddc2d94ceb3de89233f58826774bd7ce7", "subdir": "re2-2022-04-01", "mirrors": [ "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/re2/archive/2022-04-01.tar.gz" ] }, "target_root": "import targets", "target_file_name": "TARGETS.re2", "bindings": { "rules": "rules-re2" } }, "com_github_cares_cares": { "repository": { "type": "archive", "content": "4237a53f7a5dc1cfb4d04a6b6374f8674c32e271", "fetch": "https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz", "sha256": "321700399b72ed0e037d0074c629e7741f6b2ec2dda92956abe3e9671d3e268e", "sha512": "466a94efda626e815a6ef7a890637056339f883d549ea6055e289fd8cd2391130e5682c905c0fb3bd7e955af7f6deb793562c170eb0ee066a4a62085a82ba470", "subdir": "c-ares-1.19.1" }, "target_root": "import targets", "target_file_name": "TARGETS.cares", "bindings": { "rules": "rules", "grpc": "com_github_grpc_grpc" } }, "com_github_grpc_grpc": { "repository": { "type": "archive", "content": "87c37bc5be5d6ded368667d02958d488b94d6143", "fetch": "https://github.com/grpc/grpc/archive/refs/tags/v1.70.2.tar.gz", "sha256": "92f240f7267ed6cd8ba2be4c59a3b5b6ec0c4b4c466071b1e1d62165b25acf64", "sha512": "4afd34b36b4d7c41260abad6d921154a3e7176a7a46de6ae97335697db5d11f61e37b08889c007463fc8f3e800788585f5f7d5b2cec12f89b69fc9bc15efd17f", "subdir": "grpc-1.70.2", "pragma": { "special": "ignore" } }, "target_root": "import targets", "target_file_name": "TARGETS.grpc", "rule_file_name": "RULES.grpc", "bindings": { "rules": "rules-grpc", "protobuf": "protobuf", "ssl": "ssl", "absl": "com_google_absl", "zlib": "zlib", "re2": "re2", "cares": "com_github_cares_cares", "google_apis": "google_apis" }, "pkg_bootstrap": { "copy": [ "bin/grpc_cpp_plugin", "include/grpc", "include/grpcpp", "include/grpc++" ] } }, "com_github_libgit2_libgit2": { "repository": { "type": "archive", "content": "80f999c62e8da8da7d64fe543c8eeb9953bf6974", "fetch": "https://github.com/libgit2/libgit2/archive/refs/tags/v1.7.2.tar.gz", "sha256": "de384e29d7efc9330c6cdb126ebf88342b5025d920dcb7c645defad85195ea7f", "sha512": "825737e4a1991fba50ea535f15b0e560ebe76ead752e04aeba36925b944d0da77fe9826a70980a1aa3d0bf9afbedfab79dd92e799c9252931384c89ebec9b012", "subdir": "libgit2-1.7.2" }, "target_root": "import targets", "target_file_name": "TARGETS.git2", "bindings": { "rules": "rules-git2", "zlib": "zlib", "ssl": "ssl" }, "bootstrap": { "include_dir": "include", "include_name": "." }, "pkg_bootstrap": { "copy": [ "git2.h", "git2" ] } }, "catch2": { "repository": { "type": "archive", "content": "5af653af0c54ad580176a8cadb5a1cef8c880372", "fetch": "https://github.com/catchorg/Catch2/archive/refs/tags/v3.5.3.tar.gz", "sha256": "8d723b0535c94860ef8cf6231580fa47d67a3416757ecb10639e40d748ab6c71", "sha512": "57c996f679cbad212cb0fde39e506bade37bd559c0e93e20f407f2a2f029e98b78661e10257f9c8e4cb5fd7d52d0ea1eae3d4a1f989c6d66fcb281e32e1688f6", "subdir": "Catch2-3.5.3" }, "target_root": "import targets", "target_file_name": "TARGETS.catch2", "bindings": { "rules": "rules" }, "bootstrap": { "drop": true } }, "com_github_curl_curl": { "repository": { "type": "archive", "content": "7b7ace4d7cc7d4059163bbd789ef7071e5d326e5", "fetch": "https://github.com/curl/curl/releases/download/curl-8_6_0/curl-8.6.0.tar.gz", "sha256": "9c6db808160015f30f3c656c0dec125feb9dc00753596bf858a272b5dd8dc398", "sha512": "43fdb6b81b394f3382d353d5f57673b2b3d26cfe34b25d08a526bc0597f508d5298e5a7088d0736d1f139cad19cd922affa51533c3a5a4bb5f2de68891c2958d", "subdir": "curl-8.6.0" }, "target_root": "import targets", "target_file_name": "TARGETS.curl", "bindings": { "rules": "rules-curl", "zlib": "zlib", "ssl": "ssl", "cares": "com_github_cares_cares" } }, "com_github_libarchive_libarchive": { "repository": { "type": "archive", "content": "994435922d1ce63b52f6420f90b1b2a9f6670c39", "fetch": "https://github.com/libarchive/libarchive/releases/download/v3.7.7/libarchive-3.7.7.tar.gz", "sha256": "4cc540a3e9a1eebdefa1045d2e4184831100667e6d7d5b315bb1cbc951f8ddff", "sha512": "cce6eecfcd33d228bd1b1162a90bad63750adb53ac4edcaed34e2fdc30b6ba211cf1fd25d4b8761373949ceec266478b09bd70ffa4e374803a29e8573d6d149e", "subdir": "libarchive-3.7.7" }, "target_root": "import targets", "target_file_name": "TARGETS.archive", "bindings": { "rules": "rules-archive", "zlib": "zlib", "ssl": "ssl", "bzip2": "bzip2", "lzma": "lzma" } }, "repo": { "repository": { "type": "file", "path": ".", "pragma": { "to_git": true } } }, "format-json infra": { "repository": { "type": "file", "path": "format-json", "pragma": { "to_git": true } } }, "repo tree": { "repository": { "type": "tree structure", "repo": "repo" }, "target_file_name": "TARGETS.tasks", "target_root": "format-json infra", "bindings": { "format": "format-json infra" } }, "format-json/tasks": { "repository": { "type": "computed", "repo": "repo tree", "target": [ "", "" ] } }, "format-json/bin": { "repository": { "type": "file", "path": "bin", "pragma": { "to_git": true } }, "target_root": "format-json infra" }, "format-json": { "repository": { "type": "file", "path": "." }, "target_root": "format-json/tasks", "rule_root": "format-json infra", "bindings": { "bin": "format-json/bin" } }, "clang-18": { "repository": "clang-18/toolchains", "target_file_name": "clang.TARGETS", "bindings": { "rules": "clang-18/rules", "clang": "clang-18/compilers/clang-18.1.8-native" } }, "clang-18/cmake-3.27.1": { "repository": "clang-18/toolchains", "target_file_name": "cmake.TARGETS", "bindings": { "rules": "clang-18/rules", "cmake": "clang-18/tools/cmake-3.27.1" } }, "clang-18/compilers/clang-18.1.8-native": { "repository": { "type": "archive", "content": "3d2972e91936ce5d4176802eb8b5ef3e0a727914", "fetch": "https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-18.1.8.tar.gz", "sha256": "09c08693a9afd6236f27a2ebae62cda656eba19021ef3f94d59e931d662d4856", "subdir": "llvm-project-llvmorg-18.1.8", "pragma": { "special": "resolve-partially" } }, "target_root": "clang-18/compilers", "target_file_name": "clang-18-native.TARGETS", "bindings": { "rules": "clang-18/rules/clang_with_gcc13", "binutils": "clang-18/imports/binutils-latest", "patches": "clang-18/patches", "iwyu": "clang-18/iwyu-18" } }, "clang-18/compilers/gcc-13.3.0-native": { "repository": { "type": "archive", "content": "d8f1f6dac9c58f38092c84be1627547edaaa1af9", "fetch": "https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.gz", "sha256": "3a2b10cab86e32358fdac871546d57e2700e9bdb5875ef33fff5b601265b9e32", "subdir": "gcc-13.3.0", "mirrors": [ "https://ftp.fau.de/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.gz" ] }, "target_root": "clang-18/compilers", "target_file_name": "gcc-13-native.TARGETS", "bindings": { "rules": "clang-18/rules/gcc", "binutils": "clang-18/imports/binutils-latest", "gmp": "clang-18/imports/gmp-6.3.0", "mpc": "clang-18/imports/mpc-1.3.1", "mpfr": "clang-18/imports/mpfr-4.2.1", "patches": "clang-18/patches" } }, "clang-18/compilers/gcc-14.2.0-musl": { "repository": "clang-18/compilers", "target_file_name": "gcc-14-musl.TARGETS", "bindings": { "rules": "clang-18/rules/gcc", "musl-cross-make": "clang-18/imports/musl-cross-make-fe915821", "binutils": "clang-18/imports/binutils-latest", "musl": "clang-18/imports/musl-1.2.4", "gmp": "clang-18/imports/gmp-6.3.0", "mpc": "clang-18/imports/mpc-1.3.1", "mpfr": "clang-18/imports/mpfr-4.2.1", "gcc": "clang-18/compilers/gcc-14.2.0-native", "mimalloc": "clang-18/imports/mimalloc-2.1.2" } }, "clang-18/compilers/gcc-14.2.0-native": { "repository": { "type": "archive", "content": "97ed2ad91801278cb7d89c720ee1dce6fa3e2a5d", "fetch": "https://ftp.gnu.org/gnu/gcc/gcc-14.2.0/gcc-14.2.0.tar.xz", "sha256": "a7b39bc69cbf9e25826c5a60ab26477001f7c08d85cec04bc0e29cabed6f3cc9", "subdir": "gcc-14.2.0", "mirrors": [ "https://ftp.fau.de/gnu/gcc/gcc-14.2.0/gcc-14.2.0.tar.xz" ] }, "target_root": "clang-18/compilers", "target_file_name": "gcc-14-native.TARGETS", "bindings": { "rules": "clang-18/rules/gcc", "binutils": "clang-18/imports/binutils-latest", "gmp": "clang-18/imports/gmp-6.3.0", "mpc": "clang-18/imports/mpc-1.3.1", "mpfr": "clang-18/imports/mpfr-4.2.1", "patches": "clang-18/patches" } }, "clang-18/imports/binutils-latest": { "repository": { "type": "archive", "content": "17cfb0cec609e1771ed867b2d005492329759e39", "fetch": "https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz", "sha256": "ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450", "subdir": "binutils-2.41", "mirrors": [ "https://ftp.fau.de/gnu/binutils/binutils-2.41.tar.xz" ] }, "target_root": "clang-18/imports", "target_file_name": "binutils.TARGETS", "bindings": { "rules": "clang-18/rules/gcc" } }, "clang-18/imports/boringssl": { "repository": { "type": "archive", "content": "19cdde8ba529848172c09e84e3deb2c92dc670c3", "fetch": "https://github.com/google/boringssl/archive/6195bf8242156c9a2fa75702eee058f91b86a88b.tar.gz", "sha256": "ad0b806b6c5cbd6cae121c608945d5fed468748e330632e8d53315089ad52c67", "subdir": "boringssl-6195bf8242156c9a2fa75702eee058f91b86a88b" }, "target_root": "clang-18/imports", "target_file_name": "boringssl.TARGETS", "bindings": { "rules": "clang-18/rules/tools" } }, "clang-18/imports/config-3d5db9e": { "repository": { "type": "archive", "content": "3a01a6bdf61b589ad70e35e1abf15758f6c2aa39", "fetch": "https://gitweb.git.savannah.gnu.org/gitweb/?p=config.git;a=snapshot;h=3d5db9ebe8607382d17d60faf8853c944fc5f353;sf=tgz", "sha256": "b9974284ff6f9e285c8c57f57a9b0726f48576c61edc2e94a71815198fda0827", "subdir": "config-3d5db9e" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/gmp-6.3.0": { "repository": { "type": "archive", "content": "eed1334cca024677702c3a4de194758cb1b15c36", "fetch": "https://gmplib.org/download/gmp/gmp-6.3.0.tar.xz", "sha256": "a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898", "subdir": "gmp-6.3.0" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/linux-headers-4.19.88-1": { "repository": { "type": "archive", "content": "d67e9625ed2750a32eadc8abdd4a356f429e1e61", "fetch": "https://github.com/sabotage-linux/kernel-headers/releases/download/v4.19.88-1/linux-headers-4.19.88-1.tar.xz", "sha256": "995bc76ccf0c40d752b5ea67c022232a17eef6c9ec80ea74ea742e3c19992813", "subdir": "linux-headers-4.19.88-1", "pragma": { "special": "resolve-partially" } }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/mimalloc-2.1.2": { "repository": { "type": "archive", "content": "da9d9ced476e35074380f3e240b59970dc443cd8", "fetch": "https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.2.tar.gz", "sha256": "2b1bff6f717f9725c70bf8d79e4786da13de8a270059e4ba0bdd262ae7be46eb", "subdir": "mimalloc-2.1.2" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/mpc-1.3.1": { "repository": { "type": "archive", "content": "afd933ab8b5246004dc0767bcd3d51333d5ca720", "fetch": "https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz", "sha256": "ab642492f5cf882b74aa0cb730cd410a81edcdbec895183ce930e706c1c759b8", "subdir": "mpc-1.3.1", "mirrors": [ "https://ftp.fau.de/gnu/mpc/mpc-1.3.1.tar.gz" ] }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/mpfr-4.2.1": { "repository": { "type": "archive", "content": "62fb991131420c31d94e7e992c9ba798e8de866f", "fetch": "https://www.mpfr.org/mpfr-4.2.1/mpfr-4.2.1.tar.xz", "sha256": "277807353a6726978996945af13e52829e3abd7a9a5b7fb2793894e18f1fcbb2", "subdir": "mpfr-4.2.1" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/musl-1.2.4": { "repository": { "type": "archive", "content": "3ae819b834bf1ca20cff02cb67b7526372d58bfd", "fetch": "https://musl.libc.org/releases/musl-1.2.4.tar.gz", "sha256": "7a35eae33d5372a7c0da1188de798726f68825513b7ae3ebe97aaaa52114f039", "subdir": "musl-1.2.4" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/musl-cross-make-fe915821": { "repository": { "type": "archive", "content": "99cdef1bbcaf683b9801453fc78aa0c59636f9a2", "fetch": "https://github.com/richfelker/musl-cross-make/archive/fe915821b652a7fa37b34a596f47d8e20bc72338.tar.gz", "sha256": "c5df9afd5efd41c97fc7f3866664ef0c91af0ff65116e27cd9cba078c7ab33ae", "subdir": "musl-cross-make-fe915821b652a7fa37b34a596f47d8e20bc72338" }, "target_root": "clang-18/imports", "target_file_name": "musl-cross-make-fe915821.TARGETS", "bindings": { "rules": "clang-18/rules/gcc", "linux-headers": "clang-18/imports/linux-headers-4.19.88-1", "config": "clang-18/imports/config-3d5db9e", "patches": "clang-18/patches" } }, "clang-18/imports/stage-0/gmp-4.2.4": { "repository": { "type": "archive", "content": "c3b7c9fa7ff0f6634dfde7ba8bf4a9afa0b3b509", "fetch": "https://ftp.gnu.org/gnu/gmp/gmp-4.2.4.tar.gz", "sha256": "09652b51e348ea459f121c01b4b7189821e06bf457fbd85382aa6f0b741b4e78", "subdir": "gmp-4.2.4", "mirrors": [ "https://ftp.fau.de/gnu/gmp/gmp-4.2.4.tar.gz" ] }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/stage-0/mpc-0.8.1": { "repository": { "type": "archive", "content": "4828dd699fe92fc23456a7abe58fae1aa45c9e84", "fetch": "https://gcc.gnu.org/pub/gcc/infrastructure/mpc-0.8.1.tar.gz", "sha256": "e664603757251fd8a352848276497a4c79b7f8b21fd8aedd5cc0598a38fee3e4", "subdir": "mpc-0.8.1" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/stage-0/mpfr-2.3.1": { "repository": { "type": "archive", "content": "34118d1fdb9a271925d769458ededef8c595876f", "fetch": "https://www.mpfr.org/mpfr-2.3.1/mpfr-2.3.1.tar.gz", "sha256": "504e34cace2fe0ba2824abb66928b623965d6f5f9bc931316e6785db7ef2e790", "subdir": "mpfr-2.3.1" }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/stage-1/gmp-5.1.3": { "repository": { "type": "archive", "content": "ba6992490775f21d0926ab17b51a7b8b6ed5e586", "fetch": "https://ftp.gnu.org/gnu/gmp/gmp-5.1.3.tar.gz", "sha256": "71f37fe18b7eaffd0700c0d3c5062268c3933c7100c29f944b81d2b6e9f78527", "subdir": "gmp-5.1.3", "mirrors": [ "https://ftp.fau.de/gnu/gmp/gmp-5.1.3.tar.gz" ] }, "target_root": "clang-18/imports", "target_file_name": "files.TARGETS" }, "clang-18/imports/zlib": { "repository": { "type": "archive", "content": "d115fc690fa59d3be51b3442158876536140b6c2", "fetch": "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz", "sha256": "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", "subdir": "zlib-1.2.13" }, "target_root": "clang-18/imports", "target_file_name": "zlib.TARGETS", "bindings": { "rules": "clang-18/rules/tools" } }, "clang-18/iwyu-18": { "repository": { "type": "archive", "content": "04ffb3d3e78e6d20fd0ea2fe627f83de4418528d", "fetch": "https://github.com/include-what-you-use/include-what-you-use/archive/refs/tags/0.22.tar.gz", "sha256": "34c7636da2abe7b86580b53b762f5269e71efff460f24f17d5913c56eb99cb7c", "subdir": "include-what-you-use-0.22" }, "target_root": "clang-18/imports", "target_file_name": "iwyu.TARGETS" }, "clang-18/patches": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "etc/patches" } }, "clang-18/patches/busybox-1593": { "repository": { "type": "foreign file", "content": "5c8fd3526a514ff64fd3d31b9a849d0649e0d197", "fetch": "https://bugs.busybox.net/attachment.cgi?id=9751", "sha256": "6671a12c48dbcefb653fc8403d1f103a1e2eba4a49b1ee9a9c27da8aa2db80d4", "name": "remove-cbq.patch" }, "target_root": "clang-18/imports", "target_file_name": "empty.TARGETS" }, "clang-18/python-3.12.0": { "repository": "clang-18/toolchains", "target_file_name": "python.TARGETS", "bindings": { "rules": "clang-18/rules", "python": "clang-18/tools/python-3.12.0" } }, "clang-18/rules": { "repository": { "type": "zip", "content": "e046bb9eb2db84e425ad93deb275800b6be4d3a1", "fetch": "https://github.com/just-buildsystem/rules-cc/archive/refs/tags/v1.6.0-alpha+20250606.zip", "sha256": "cb544ea41aec99e14ce874ac968ae4d21c3af4ce3de8cfd692a24b6baa2c3003", "subdir": "rules-cc-1.6.0-alpha-20250606/rules" } }, "clang-18/rules/clang_with_gcc13": { "repository": "clang-18/rules", "target_root": "clang-18/defaults", "rule_root": "clang-18/rules", "target_file_name": "clang.TARGETS", "bindings": { "base": "clang-18/rules/gcc", "gcc": "clang-18/compilers/gcc-13.3.0-native", "cmake": "clang-18/cmake-3.27.1", "python": "clang-18/python-3.12.0", "stage-0": "clang-18/rules/stage-0" } }, "clang-18/rules/gcc": { "repository": "clang-18/rules", "target_root": "clang-18/defaults", "rule_root": "clang-18/rules", "target_file_name": "gcc.TARGETS", "bindings": { "base": "clang-18/rules/stage-1", "gcc": "clang-18/stage-1/gcc", "stage-0": "clang-18/rules/stage-0" } }, "clang-18/rules/stage-0": { "repository": "clang-18/rules", "target_root": "clang-18/defaults", "rule_root": "clang-18/rules", "target_file_name": "stage-0.TARGETS", "bindings": { "busybox": "clang-18/stage-0/busybox" } }, "clang-18/rules/stage-1": { "repository": "clang-18/rules", "target_root": "clang-18/defaults", "rule_root": "clang-18/rules", "target_file_name": "stage-1.TARGETS", "bindings": { "make": "clang-18/stage-0/make", "busybox": "clang-18/stage-0/busybox", "gcc": "clang-18/stage-0/gcc", "stage-0": "clang-18/rules/stage-0" } }, "clang-18/rules/tools": { "repository": "clang-18/rules", "target_root": "clang-18/defaults", "rule_root": "clang-18/rules", "target_file_name": "tools.TARGETS", "bindings": { "base": "clang-18/rules/gcc", "gcc-musl": "clang-18/compilers/gcc-14.2.0-musl", "stage-0": "clang-18/rules/stage-0" } }, "clang-18/stage-0/binutils": { "repository": "clang-18/imports/binutils-latest", "target_root": "clang-18/bootstrap", "target_file_name": "stage-0-binutils.TARGETS", "bindings": { "rules": "clang-18/rules/stage-0", "busybox": "clang-18/stage-0/busybox", "make": "clang-18/stage-0/make" } }, "clang-18/stage-0/busybox": { "repository": { "type": "archive", "content": "529defd1de4d1e362458e6561017ae74b2b3f28e", "fetch": "https://busybox.net/downloads/busybox-1.36.1.tar.bz2", "sha256": "b8cc24c9574d809e7279c3be349795c5d5ceb6fdf19ca709f80cde50e47de314", "subdir": "busybox-1.36.1" }, "target_root": "clang-18/bootstrap", "target_file_name": "stage-0-busybox.TARGETS", "bindings": { "rules": "clang-18/rules/stage-0", "gcc": "clang-18/stage-0/gcc", "make": "clang-18/stage-0/make", "patch-cbq": "clang-18/patches/busybox-1593" } }, "clang-18/stage-0/gcc": { "repository": { "type": "archive", "content": "4819a9afa95dae6ac8d5abec15049a66e3e725a0", "fetch": "https://ftp.gnu.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.gz", "sha256": "ddbaa583c5d4e4f0928bf15d9f6b6c283349e16eedc47bde71e1b813f6f37819", "subdir": "gcc-4.7.4", "mirrors": [ "https://ftp.fau.de/gnu//gcc/gcc-4.7.4/gcc-4.7.4.tar.gz" ] }, "target_root": "clang-18/bootstrap", "target_file_name": "stage-0-gcc.TARGETS", "bindings": { "rules": "clang-18/rules/stage-0", "busybox": "clang-18/stage-0/busybox", "make": "clang-18/stage-0/make", "binutils": "clang-18/stage-0/binutils", "gmp": "clang-18/imports/stage-0/gmp-4.2.4", "mpc": "clang-18/imports/stage-0/mpc-0.8.1", "mpfr": "clang-18/imports/stage-0/mpfr-2.3.1", "patches": "clang-18/patches" } }, "clang-18/stage-0/make": { "repository": { "type": "archive", "content": "4adc00a78258ae2eb53d103ef2c1ecf291a86fbf", "fetch": "https://ftp.gnu.org/gnu/make/make-4.4.1.tar.gz", "sha256": "dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3", "subdir": "make-4.4.1", "mirrors": [ "https://ftp.fau.de/gnu/make/make-4.4.1.tar.gz" ] }, "target_root": "clang-18/bootstrap", "target_file_name": "stage-0-make.TARGETS", "bindings": { "rules": "clang-18/rules/stage-0", "busybox": "clang-18/stage-0/busybox", "gcc": "clang-18/stage-0/gcc" } }, "clang-18/stage-1/gcc": { "repository": { "type": "archive", "content": "25e1bc0b9c97916a3e6c3f4c100bf170ddf06eaa", "fetch": "https://ftp.gnu.org/gnu/gcc/gcc-10.2.0/gcc-10.2.0.tar.gz", "sha256": "27e879dccc639cd7b0cc08ed575c1669492579529b53c9ff27b0b96265fa867d", "subdir": "gcc-10.2.0", "mirrors": [ "https://ftp.fau.de/gnu/gcc/gcc-10.2.0/gcc-10.2.0.tar.gz" ] }, "target_root": "clang-18/bootstrap", "target_file_name": "stage-1-gcc.TARGETS", "bindings": { "rules": "clang-18/rules/stage-1", "gmp": "clang-18/imports/stage-1/gmp-5.1.3", "mpc": "clang-18/imports/mpc-1.3.1", "mpfr": "clang-18/imports/mpfr-4.2.1", "binutils": "clang-18/imports/binutils-latest", "patches": "clang-18/patches" } }, "clang-18/tools/cmake-3.27.1": { "repository": { "type": "archive", "content": "45586697d7bb7d4f3cae4c86bba5bde710a367a9", "fetch": "https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1.tar.gz", "sha256": "b1a6b0135fa11b94476e90f5b32c4c8fad480bf91cf22d0ded98ce22c5132004", "subdir": "cmake-3.27.1" }, "target_root": "clang-18/tools", "target_file_name": "cmake-3.27.TARGETS", "bindings": { "rules": "clang-18/rules/tools", "ssl": "clang-18/imports/boringssl" } }, "clang-18/tools/python-3.12.0": { "repository": { "type": "archive", "content": "0b9a01c1b77e8b75a977e7e8e447d6764215eb1b", "fetch": "https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tar.xz", "sha256": "795c34f44df45a0e9b9710c8c71c15c671871524cd412ca14def212e8ccb155d", "subdir": "Python-3.12.0" }, "target_root": "clang-18/tools", "target_file_name": "python-3.12.TARGETS", "bindings": { "rules": "clang-18/rules/tools", "zlib": "clang-18/imports/zlib" } }, "clang-18/bootstrap": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "src/bootstrap" } }, "clang-18/compilers": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "src/compilers" } }, "clang-18/defaults": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "etc/defaults" } }, "clang-18/imports": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "etc/imports" } }, "clang-18/toolchains": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "toolchains" } }, "clang-18/tools": { "repository": { "type": "git", "repository": "https://github.com/just-buildsystem/bootstrappable-toolchain", "branch": "master", "commit": "538beb34980293352b774ead73ec483ca6a211c7", "subdir": "src/tools" } } } }just-buildsystem-justbuild-b1fb5fa/etc/toolchain/000077500000000000000000000000001516554100600223525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/toolchain/CC/000077500000000000000000000000001516554100600226375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/toolchain/CC/TARGETS000066400000000000000000000164011516554100600236750ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "arguments_config": ["OS", "ARCH", "TOOLCHAIN_CONFIG"] , "base": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" , { "type": "if" , "cond": { "type": "and" , "$1": [{"type": "var", "name": "OS"}, {"type": "var", "name": "ARCH"}] } , "then": { "type": "lookup" , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , "key": "FAMILY" } , "else": "unknown" } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} , "case": {"gnu": ["gcc"], "clang": ["clang"], "unknown": ["unknown"]} , "default": { "type": "fail" , "msg": [ "Unsupported TOOLCHAIN_CONFIG[FAMILY]" , {"type": "var", "name": "COMPILER_FAMILY"} ] } } } , "ADD_LDFLAGS": { "type": "lookup" , "key": "ADD_LDFLAGS" , "default": [] , "map": { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } } } , "unknown": { "type": ["CC", "defaults"] , "CC": ["cc"] , "CXX": ["c++"] , "AR": ["ar"] , "DWP": ["dwp"] , "PATH": ["/bin", "/sbin", "/usr/bin", "/usr/sbin"] } , "gcc": { "type": ["CC", "defaults"] , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "CC": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["i686-linux-gnu-gcc"] , "linux_x86_64": ["x86_64-linux-gnu-gcc"] , "linux_arm": ["arm-linux-gnueabi-gcc"] , "linux_arm64": ["aarch64-linux-gnu-gcc"] } , "default": { "type": "fail" , "msg": ["Unsupported PLATFORM for gcc", {"type": "var", "name": "PLATFORM"}] } } } , "CXX": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["i686-linux-gnu-g++"] , "linux_x86_64": ["x86_64-linux-gnu-g++"] , "linux_arm": ["arm-linux-gnueabi-g++"] , "linux_arm64": ["aarch64-linux-gnu-g++"] } , "default": { "type": "fail" , "msg": ["Unsupported PLATFORM for g++", {"type": "var", "name": "PLATFORM"}] } } } , "CXXFLAGS": { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } , "case": {"arm": ["-Wno-psabi"]} } , "AR": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["i686-linux-gnu-ar"] , "linux_x86_64": ["x86_64-linux-gnu-ar"] , "linux_arm": ["arm-linux-gnueabi-ar"] , "linux_arm64": ["aarch64-linux-gnu-ar"] } , "default": { "type": "fail" , "msg": ["Unsupported PLATFORM for ar", {"type": "var", "name": "PLATFORM"}] } } } , "PATH": ["/bin", "/sbin", "/usr/bin", "/usr/sbin"] } , "clang": { "type": ["CC", "defaults"] , "arguments_config": ["OS", "ARCH", "TARGET_ARCH", "DEBUG"] , "CC": ["clang"] , "CXX": ["clang++"] , "AR": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["i686-linux-gnu-ar"] , "linux_x86_64": ["x86_64-linux-gnu-ar"] , "linux_arm": ["arm-linux-gnueabi-ar"] , "linux_arm64": ["aarch64-linux-gnu-ar"] } , "default": { "type": "fail" , "msg": ["Unsupported PLATFORM for ar", {"type": "var", "name": "PLATFORM"}] } } } , "CFLAGS": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["-target", "i686-linux-gnu"] , "linux_x86_64": ["-target", "x86_64-linux-gnu"] , "linux_arm": ["-target", "arm-linux-gnueabi"] , "linux_arm64": ["-target", "aarch64-linux-gnu"] } , "default": { "type": "fail" , "msg": [ "Unsupported PLATFORM for clang" , {"type": "var", "name": "PLATFORM"} ] } } } , "CXXFLAGS": { "type": "let*" , "bindings": [ [ "PLATFORM" , { "type": "join" , "separator": "_" , "$1": [ {"type": "var", "name": "OS"} , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] } ] ] , "body": { "type": "case" , "expr": {"type": "var", "name": "PLATFORM"} , "case": { "linux_x86": ["-target", "i686-linux-gnu"] , "linux_x86_64": ["-target", "x86_64-linux-gnu"] , "linux_arm": ["-target", "arm-linux-gnueabi"] , "linux_arm64": ["-target", "aarch64-linux-gnu"] } , "default": { "type": "fail" , "msg": [ "Unsupported PLATFORM for clang++" , {"type": "var", "name": "PLATFORM"} ] } } } , "ADD_COMPILE_FLAGS": { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-fdebug-compilation-dir=."] } , "PATH": ["/bin", "/sbin", "/usr/bin", "/usr/sbin"] } } just-buildsystem-justbuild-b1fb5fa/etc/toolchain/patch/000077500000000000000000000000001516554100600234515ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/toolchain/patch/TARGETS000066400000000000000000000001561516554100600245070ustar00rootroot00000000000000{ "defaults": { "type": ["patch", "defaults"] , "PATCH": ["patch"] , "PATH": ["/bin", "/usr/bin"] } } just-buildsystem-justbuild-b1fb5fa/etc/toolchain/shell/000077500000000000000000000000001516554100600234615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/etc/toolchain/shell/TARGETS000066400000000000000000000000431516554100600245120ustar00rootroot00000000000000{"defaults": {"type": "defaults"}} just-buildsystem-justbuild-b1fb5fa/format-json/000077500000000000000000000000001516554100600220565ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/format-json/EXPRESSIONS000066400000000000000000000025051516554100600236250ustar00rootroot00000000000000{ "stage_singleton_field": { "vars": ["fieldname", "transition", "location"] , "expression": { "type": "assert_non_empty" , "msg": ["No artifact specified in field", {"type": "var", "name": "fieldname"}] , "$1": { "type": "disjoint_map_union" , "msg": [ "Expecting (essentially) a single artifact in field" , {"type": "var", "name": "fieldname"} ] , "$1": { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "artifact" , "range": { "type": "values" , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "src"} , "transition": { "type": "var" , "name": "transition" , "default": {"type": "empty_map"} } } } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "location"} , "value": {"type": "var", "name": "artifact"} } } } } } } } } just-buildsystem-justbuild-b1fb5fa/format-json/RULES000066400000000000000000000102731516554100600226760ustar00rootroot00000000000000{ "fmt": { "target_fields": ["files"] , "implicit": {"formatter": [["@", "bin", "", "json-format.py"]]} , "imports": {"stage": "stage_singleton_field"} , "tainted": ["lint"] , "expression": { "type": "let*" , "bindings": [ ["fieldname", "formatter"] , ["location", "format"] , ["format", {"type": "CALL_EXPRESSION", "name": "stage"}] , [ "diffs" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "files"} , "body": { "type": "disjoint_map_union" , "$1": { "type": "foreach_map" , "range": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "dep"} } , "body": { "type": "ACTION" , "outs": [ { "type": "join" , "$1": [{"type": "var", "name": "_"}, ".diff"] } ] , "inputs": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "format"} , { "type": "singleton_map" , "key": { "type": "join" , "$1": [{"type": "var", "name": "_"}, ".orig"] } , "value": {"type": "var", "name": "$_"} } ] } , "cmd": [ "sh" , "-c" , { "type": "join" , "separator": " " , "$1": [ "./format -s" , { "type": "join_cmd" , "$1": [ { "type": "join" , "$1": [{"type": "var", "name": "_"}, ".orig"] } ] } , ">" , { "type": "join_cmd" , "$1": [{"type": "var", "name": "_"}] } , "&& diff -u" , { "type": "join_cmd" , "$1": [ { "type": "join" , "$1": [{"type": "var", "name": "_"}, ".orig"] } ] } , { "type": "join_cmd" , "$1": [{"type": "var", "name": "_"}] } , ">" , { "type": "join_cmd" , "$1": [ { "type": "join" , "$1": [{"type": "var", "name": "_"}, ".diff"] } ] } ] } ] , "may_fail": ["lint"] , "fail_message": { "type": "join" , "$1": [ "Target file " , {"type": "var", "name": "_"} , " not formatted correctly." ] } } } } } } ] , [ "diff" , { "type": "ACTION" , "inputs": {"type": "var", "name": "diffs"} , "outs": ["targets.diff"] , "cmd": [ "sh" , "-c" , { "type": "join" , "separator": " " , "$1": [ "cat" , { "type": "join_cmd" , "$1": {"type": "keys", "$1": {"type": "var", "name": "diffs"}} } , "> targets.diff" ] } ] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "diff"}} } } } just-buildsystem-justbuild-b1fb5fa/format-json/TARGETS000066400000000000000000000000031516554100600231030ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/format-json/TARGETS.tasks000066400000000000000000000006441516554100600242420ustar00rootroot00000000000000{ "ls": { "type": "generic" , "outs": ["json-files"] , "cmds": [ "find . '(' -name 'TARGETS*' -o -name RULES -o -name EXPRESSIONS ')' -type f > json-files" ] , "deps": [["TREE", null, "."]] } , "targets": { "type": "generic" , "outs": ["TARGETS"] , "cmds": ["python3 generate-fmt.py"] , "deps": ["ls", ["@", "format", "", "generate-fmt.py"]] } , "": {"type": "export", "target": "targets"} } just-buildsystem-justbuild-b1fb5fa/format-json/generate-fmt.py000066400000000000000000000015341516554100600250110ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json with open("json-files") as f: tfiles = f.read().splitlines() targets = {"": {"type": "fmt", "files": [t.removeprefix("./") for t in tfiles]}} with open("TARGETS", "w") as f: json.dump(targets, f, indent=2) just-buildsystem-justbuild-b1fb5fa/lint/000077500000000000000000000000001516554100600205655ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/lint/TARGETS000066400000000000000000000057221516554100600216270ustar00rootroot00000000000000{ "": { "type": "install" , "tainted": ["lint", "test"] , "deps": ["LINT"] , "files": {"REPORT": "lint-report", "format.diff": "format.diff"} } , "LINT": { "type": "install" , "tainted": ["lint", "test"] , "dirs": [ ["LINT: clang-tidy", "clang-tidy"] , ["LINT: clang-format", "clang-format"] , ["LINT: iwyu", "iwyu"] , ["LINT: strict_deps", "strict_deps"] ] } , "clang toolchain": { "type": "configure" , "arguments_config": ["TOOLCHAIN_CONFIG"] , "target": ["@", "clang", "", "toolchain"] , "config": { "type": "`" , "$1": { "TOOLCHAIN_CONFIG": { "type": "," , "$1": { "type": "map_union" , "$1": [ { "type": "var" , "name": "TOOLCHAIN_CONFIG" , "default": {"type": "empty_map"} } , {"type": "'", "$1": {"INCLUDE_LINTER": true}} ] } } } } } , "clang": {"type": "install", "dirs": [["clang toolchain", "toolchain"]]} , "LINT: clang-tidy": { "type": ["@", "rules", "lint", "targets"] , "tainted": ["test"] , "name": ["clang-tidy"] , "linter": ["run_clang_tidy.py"] , "summarizer": ["summary.py"] , "config": [["@", "src", "", ".clang-tidy"], "clang"] , "targets": [ ["@", "src", "src/buildtool/main", "just"] , ["@", "src", "src/other_tools/just_mr", "just-mr"] , ["@", "tests", "", "ALL"] ] } , "LINT: clang-format": { "type": ["@", "rules", "lint", "targets"] , "tainted": ["test"] , "name": ["clang-format"] , "linter": ["run_clang_format.py"] , "summarizer": ["summary.py"] , "config": [["@", "src", "", ".clang-format"], "clang"] , "targets": [ ["@", "src", "src/buildtool/main", "just"] , ["@", "src", "src/other_tools/just_mr", "just-mr"] , ["@", "tests", "", "ALL"] ] } , "format.diff": { "type": "generic" , "tainted": ["lint", "test"] , "cmds": ["./create-diff.py"] , "outs": ["format.diff"] , "deps": ["LINT: clang-format", "create-diff.py"] } , "iwyu config": {"type": "install", "files": {"iwyu-mapping": "iwyu-mapping.imp"}} , "LINT: iwyu": { "type": ["@", "rules", "lint", "targets"] , "tainted": ["test"] , "name": ["iwyu"] , "linter": ["run_iwyu.py"] , "summarizer": ["summary.py"] , "config": ["iwyu config", "clang"] , "targets": [ ["@", "src", "src/buildtool/main", "just"] , ["@", "src", "src/other_tools/just_mr", "just-mr"] , ["@", "tests", "", "ALL"] ] } , "LINT: strict_deps": { "type": ["@", "rules", "lint", "targets"] , "tainted": ["test"] , "name": ["strict deps"] , "linter": ["run_strict_deps.py"] , "summarizer": ["summary.py"] , "targets": [ ["@", "src", "src/buildtool/main", "just"] , ["@", "src", "src/other_tools/just_mr", "just-mr"] , ["@", "tests", "", "ALL"] ] } , "lint-report": { "type": "generic" , "tainted": ["lint", "test"] , "cmds": ["./combined-report.py"] , "outs": ["report"] , "deps": ["LINT", "combined-report.py"] } } just-buildsystem-justbuild-b1fb5fa/lint/combined-report.py000077500000000000000000000020301516554100600242260ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os with open("report", "w") as out: for lint in sorted(os.listdir()): if os.path.isdir(lint): with open(os.path.join(lint, "report")) as f: report = f.read() if len(report) > 1: out.write(lint + "\n") out.write("".join( [" " + line + "\n" for line in report.splitlines()]) + "\n") just-buildsystem-justbuild-b1fb5fa/lint/create-diff.py000077500000000000000000000020441516554100600233130ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json with open("out/failures.json") as f: failures = json.load(f) with open("format.diff", "w") as f: for k, v in failures.items(): src = v["config"]["src"] if src.startswith("work/"): src = src[len("work/"):] diff = v["log"].splitlines() f.write("--- a/%s\n" % (src, )) f.write("+++ b/%s\n" % (src, )) for l in diff[2:]: f.write(l + "\n") just-buildsystem-justbuild-b1fb5fa/lint/iwyu-mapping.imp000066400000000000000000000171271516554100600237320ustar00rootroot00000000000000# Mapping file for include-what-you-use. # Check rules at: https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md # Take into account that the order in which entries are given for the same # symbol is important. # For instance for grpc::Status has a higher priority: # { "symbol": [ "grpc::Status", "private", "", "public"] }, # { "symbol": [ "grpc::Status", "private", "", "public"] }, [ # Map C headers { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, # Hide std implementation details headers: { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, { "include": ["", "private", "", "public"] }, # Map symbols: { "symbol": ["timespec", "private", "", "public"] }, { "symbol": ["tm", "private", "", "public"] }, { "symbol": ["AT_FDCWD", "private", "", "public"] }, { "symbol": ["SEEK_END", "private", , "public"] }, { "symbol": ["SEEK_SET", "private", , "public"] }, { "symbol": ["pid_t", "private", , "public"] }, { "symbol": ["std::size_t", "private", "", "public"] }, { "symbol": ["size_t", "private", "", "public"] }, { "symbol": ["std::nullptr_t", "private", "", "public"] }, { "symbol": ["std::ptrdiff_t", "private", "", "public"] }, { "symbol": ["FILE", "private", "", "public"] }, { "symbol": ["NULL", "private", "", "public"] }, { "symbol": ["std::thread::id", "private", "", "public"] }, { "symbol": ["uint", "private", "", "public"] }, { "symbol": ["std::uint", "private", "", "public"] }, { "symbol": ["std::hash", "private", "", "public"] }, # Trick to "ignore" std::allocator and similar types we don't want to # include manually. Taken from: # https://github.com/include-what-you-use/include-what-you-use/blob/d2d092919f2774b5463e236e1ee9d56fb46ceb60/gcc.symbols.imp { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::allocator", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "symbol": [ "std::char_traits", "private", "", "public"] }, { "include": ["", "public", "", "public"] }, # Model the dependencies between the public standard stream headers, # in order to include only the top-most library. # Taken from: # https://github.com/include-what-you-use/include-what-you-use/blob/377eaef70cdda47368939f4d9beabfabe3f628f0/iwyu_include_picker.cc#L623 { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public" ] }, { "include": [ "", "public", "", "public"] }, # Use common headers for third-parties: { "include": ["@\"gsl/.*\"", "private", "\"gsl/gsl\"", "public"] }, { "include": ["@\"nlohmann/.*\"", "private", "\"nlohmann/json.hpp\"", "public"] }, # "git2/XXX.h" => where XXX doesn't contain '/' { "include": ["@\"git2/[^/]+\\.h\"", "private", "", "public"] }, # "fmt/core.h" is for compatibility reasons; otherwise, include the suggested # headers if they don't break the pkg build { "include": ["\"fmt/format.h\"", "private", "\"fmt/core.h\"", "public"] }, { "include": ["\"fmt/base.h\"", "private", "\"fmt/core.h\"", "public"] }, { "include": ["@\"catch2/matchers/.*\"", "private", "\"catch2/matchers/catch_matchers_all.hpp\"", "public"] }, { "include": ["@\"catch2/generators/.*\"", "private", "\"catch2/generators/catch_generators_all.hpp\"", "public"] }, # For compatibility reasons map all openssl headers to sha.h { "include": ["@\"openssl/.*\"", "private", "\"openssl/sha.h\"", "public"] }, # GRPC: { "include": ["@\"grpcpp/.*\"", "private", "", "public"] }, { "include": ["@", "private", "", "public"] }, { "symbol": [ "grpc::Status", "private", "", "public"] }, { "symbol": [ "grpc::Status", "private", "", "public"] }, # bazel Digest should be included via our bazel_types header: { "include": [ "\"build/bazel/remote/execution/v2/remote_execution.pb.h\"", "private" , "\"src/buildtool/common/bazel_types.hpp\"", "public" ] }, # bazel_re namespace alias should also come from the bazel_types header: { "symbol": ["bazel_re", "private", "\"src/buildtool/common/bazel_types.hpp\"", "public"] }, ] just-buildsystem-justbuild-b1fb5fa/lint/run_clang_format.py000077500000000000000000000043331516554100600244650ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import subprocess import sys from typing import List def dump_meta(src: str, cmd: List[str]) -> None: """Dump linter action metadata for further analysis.""" OUT = os.environ.get("OUT") if OUT: with open(os.path.join(OUT, "config.json"), "w") as f: json.dump({"src": src, "cmd": cmd}, f) def run_lint(src: str, cmd: List[str]) -> int: """Run the lint command for the specified source file.""" dump_meta(src, cmd) CONFIG = os.environ.get("CONFIG") if CONFIG is None: print("Failed to get CONFIG", file=sys.stderr) return 1 shutil.copyfile(os.path.join(CONFIG, ".clang-format"), ".clang-format") extra = [] if src.endswith(".tpp"): extra = ["-x", "c++"] db = [{ "directory": os.getcwd(), "arguments": cmd[:1] + extra + cmd[1:], "file": src }] with open("compile_commands.json", "w") as f: json.dump(db, f) new_cmd = [ os.path.join(CONFIG, "toolchain", "bin", "clang-format"), "-style=file", src, ] OUT = os.environ.get("OUT") if OUT is None: print("Failed to get OUT", file=sys.stderr) sys.exit(1) formatted = os.path.join(OUT, "formatted") with open(formatted, "w") as f: print("Running cmd %r with db %r" % (new_cmd, db), file=sys.stderr) res = subprocess.run(new_cmd, stdout=f).returncode if res != 0: return res return subprocess.run(["diff", "-u", src, formatted]).returncode if __name__ == "__main__": sys.exit(run_lint(sys.argv[1], sys.argv[2:])) just-buildsystem-justbuild-b1fb5fa/lint/run_clang_tidy.py000077500000000000000000000050211516554100600241410ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import subprocess import sys from typing import List CXX_LIB_VERSION = "13.3.0" def dump_meta(src: str, cmd: List[str]) -> None: """Dump linter action metadata for further analysis.""" OUT = os.environ.get("OUT") if OUT is not None: with open(os.path.join(OUT, "config.json"), "w") as f: json.dump({"src": src, "cmd": cmd}, f) def run_lint(src: str, cmd: List[str]) -> int: """Run the lint command for the specified source file.""" dump_meta(src, cmd) CONFIG = os.environ.get("CONFIG") if CONFIG is None: print("Failed to get CONFIG", file=sys.stderr) return 1 shutil.copyfile(os.path.join(CONFIG, ".clang-tidy"), ".clang-tidy") extra = ["-Wno-unused-command-line-argument"] # add include paths from the bundled toolchain baseincludepath = os.path.join(CONFIG, "toolchain", "include", "c++", CXX_LIB_VERSION) # We're using the native toolchain, so arch-specific headers are # only available for one arch. Hence we can try all supported candidates # and add the ones found for arch in ["x86_64", "arm"]: idir = os.path.join(baseincludepath, "%s-pc-linux-gnu" % (arch, )) if os.path.exists(idir): extra += ["-isystem", idir] extra += [ "-isystem", baseincludepath, ] if src.endswith(".tpp"): extra += ["-x", "c++"] db = [{ "directory": os.getcwd(), "arguments": cmd[:1] + extra + cmd[1:], "file": src }] with open("compile_commands.json", "w") as f: json.dump(db, f) new_cmd = [ os.path.join(CONFIG, "toolchain", "bin", "clang-tidy"), src, ] print("Running cmd %r with db %r" % (new_cmd, db), file=sys.stderr) return subprocess.run(new_cmd).returncode if __name__ == "__main__": sys.exit(run_lint(sys.argv[1], sys.argv[2:])) just-buildsystem-justbuild-b1fb5fa/lint/run_iwyu.py000077500000000000000000000112221516554100600230210ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import subprocess import sys from typing import List CXX_LIB_VERSION = "13.3.0" def dump_meta(src: str, cmd: List[str]) -> None: """Dump linter action metadata for further analysis.""" OUT = os.environ.get("OUT") if OUT: with open(os.path.join(OUT, "config.json"), "w") as f: json.dump({"src": src, "cmd": cmd}, f) def run_lint(src: str, cmd: List[str]) -> int: """Run the lint command for the specified source file.""" dump_meta(src, cmd) CONFIG = os.environ.get("CONFIG") if CONFIG is None: print("Failed to get CONFIG", file=sys.stderr) return 1 shutil.copyfile(os.path.join(CONFIG, "iwyu-mapping"), "iwyu-mapping.imp") # Currently, .tpp files cannot be processed to produce meaningful suggestions # Their corresponding .hpp files are analyzed separately, so just skip if src.endswith(".tpp"): return 0 # IWYU doesn't process headers by default if src.endswith(".hpp"): # To analyze .hpp files, we need to "pack" them into a source file: # _fake.cpp includes the header and marks it as associated. # We could simply change the command line, but this may cause # IWYU to search for associated files by default (using stem). # Example: # // src/utils/cpp/gsl.hpp considered a source file: # #include "gsl/gsl" // analysis happens because of names # ... TMPDIR = os.environ.get("TMPDIR") if TMPDIR: fake_stem = os.path.join(TMPDIR, "_fake") with open(fake_stem + ".cpp", "w+") as fake: content = "#include \"%s\" // IWYU pragma: associated" % ( src.split(os.path.sep, 1)[1]) fake.write(content) src = os.path.realpath(fake.name) else: print("Failed to get TMPDIR", file=sys.stderr) return 1 # The command line must be adjusted as well e_index = cmd.index("-E") cmd = cmd[:e_index] + [ "-c", src, "-o", os.path.realpath(fake_stem) + ".o" ] + cmd[e_index + 2:] iwyu_flags: List[str] = [ "--cxx17ns", # suggest the more concise syntax introduced in C++17 "--no_fwd_decls", # do not use forward declarations "--no_default_mappings", # do not add iwyu's default mappings "--mapping_file=iwyu-mapping.imp", # specifies a mapping file "--error", # return 1 if there are any suggestions "--verbose=2", # provide explanations "--max_line_length=1000" # don't limit explanation messages ] iwyu_options: List[str] = [] for option in iwyu_flags: iwyu_options.append("-Xiwyu") iwyu_options.append(option) # add include paths from the bundled toolchain baseincludepath = os.path.join(CONFIG, "toolchain", "include", "c++", CXX_LIB_VERSION) # We're using the native toolchain, so arch-specific headers are # only available for one arch. Hence we can try all supported candidates # and add the ones found for arch in ["x86_64", "arm"]: idir = os.path.join(baseincludepath, "%s-pc-linux-gnu" % (arch, )) if os.path.exists(idir): iwyu_options += ["-isystem", idir] iwyu_options += [ "-isystem", baseincludepath, ] iwyu_options.extend(cmd[1:]) # IWYU uses <> for headers from -isystem and quoted for -I # https://github.com/include-what-you-use/include-what-you-use/issues/1070#issuecomment-1163177107 # So we "ask" IWYU here to include all non-system dependencies with quotes iwyu_options = ["-I" if x == "-isystem" else x for x in iwyu_options] new_cmd = [os.path.join(CONFIG, "toolchain", "bin", "include-what-you-use")] new_cmd.extend(iwyu_options) new_cmd.append(src) print("Running cmd %r" % (new_cmd), file=sys.stderr) return subprocess.run(new_cmd, stdout=subprocess.DEVNULL).returncode if __name__ == "__main__": sys.exit(run_lint(sys.argv[1], sys.argv[2:])) just-buildsystem-justbuild-b1fb5fa/lint/run_strict_deps.py000077500000000000000000000054311516554100600243540ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import sys from typing import List def dump_meta(src: str, cmd: List[str]) -> None: """Dump linter action metadata for further analysis.""" OUT = os.environ.get("OUT") if OUT: with open(os.path.join(OUT, "config.json"), "w") as f: json.dump({"src": src, "cmd": cmd}, f) def run_lint(src: str, cmd: List[str]) -> int: """Run the lint command for the specified source file.""" dump_meta(src, cmd) META = os.environ.get("META") if META is None: print("Failed to get META", file=sys.stderr) return 1 direct: List[str] = [] with open(META) as f: direct = json.load(f)["direct deps artifact names"] include_dirs: List[str] = [] for i in range(len(cmd)): if cmd[i] in ["-I", "-isystem"]: include_dirs += [cmd[i + 1]] with open(src) as f: lines = f.read().splitlines() failed: bool = False def include_covered(include_path: str) -> bool: for d in direct: rel_path = os.path.relpath(include_path, d) if not rel_path.startswith('../'): return True return False def handle_resolved_include(i: int, resolved: str) -> None: nonlocal failed if not include_covered(resolved): failed = True print("%03d %s" % (i, lines[i])) print( " ---> including %r which is only provided by an indirect dependency" % (resolved, )) def handle_include(i: int, to_include: str) -> None: for d in include_dirs: candidate = os.path.join(d, to_include) if os.path.exists(candidate): handle_resolved_include(i, candidate) for i in range(len(lines)): to_include = None if lines[i].startswith('#include "'): to_include = lines[i].split('"')[1] if lines[i].startswith('#include <'): to_include = lines[i].split('<', 1)[1].split('>')[0] if to_include: # if non-empty string handle_include(i, to_include) return 1 if failed else 0 if __name__ == "__main__": sys.exit(run_lint(sys.argv[1], sys.argv[2:])) just-buildsystem-justbuild-b1fb5fa/lint/summary.py000077500000000000000000000032631516554100600226430ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import sys from typing import Any, Dict FAILED: Dict[str, Any] = {} for lint in sorted(os.listdir()): if os.path.isdir(lint): with open(os.path.join(lint, "result")) as f: result = f.read().strip() if result != "PASS": record = {} with open(os.path.join(lint, "out/config.json")) as f: record["config"] = json.load(f) with open(os.path.join(lint, "stdout")) as f: log = f.read() with open(os.path.join(lint, "stderr")) as f: log += f.read() record["log"] = log FAILED[lint] = record OUT = os.environ.get("OUT") if OUT is None: print("Failed to get OUT") sys.exit(1) with open(os.path.join(OUT, "failures.json"), "w") as f: json.dump(FAILED, f) failures = list(FAILED.keys()) for f in failures: src = FAILED[f]["config"]["src"] log = FAILED[f]["log"] print("%s %s" % (f, src)) print("".join([" " + line + "\n" for line in log.splitlines()])) if failures: sys.exit(1) just-buildsystem-justbuild-b1fb5fa/rules/000077500000000000000000000000001516554100600207515ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/000077500000000000000000000000001516554100600212365ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/EXPRESSIONS000066400000000000000000002642711516554100600230170ustar00rootroot00000000000000{ "default-CC": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "CC"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["CC", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , [ "CC" , { "type": "if" , "cond": {"type": "var", "name": "CC"} , "then": {"type": "var", "name": "CC"} , "else": ["cc"] } ] ] , "body": {"type": "var", "name": "CC"} } } , "default-CXX": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "CXX"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["CXX", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , [ "CXX" , { "type": "if" , "cond": {"type": "var", "name": "CXX"} , "then": {"type": "var", "name": "CXX"} , "else": ["c++"] } ] ] , "body": {"type": "var", "name": "CXX"} } } , "default-AR": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "AR"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["AR", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , [ "AR" , { "type": "if" , "cond": {"type": "var", "name": "AR"} , "then": {"type": "var", "name": "AR"} , "else": ["ar"] } ] ] , "body": {"type": "var", "name": "AR"} } } , "default-DWP": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "DWP"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["DWP", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , [ "DWP" , { "type": "if" , "cond": {"type": "var", "name": "DWP"} , "then": {"type": "var", "name": "DWP"} , "else": ["dwp"] } ] ] , "body": {"type": "var", "name": "DWP"} } } , "default-ARFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "ARFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["ARFLAGS", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , [ "ARFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "ARFLAGS"} , "then": {"type": "var", "name": "ARFLAGS"} , "else": ["cqs"] } ] ] , "body": {"type": "var", "name": "ARFLAGS"} } } , "default-CFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "CFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-CXXFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "CXXFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-LDFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "LDFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-ENV": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "ENV"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-PATH": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "PATH"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-TOOLCHAIN": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "defaults"] , ["provider", "TOOLCHAIN"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-NON_SYSTEM_TOOLS": { "vars": ["defaults-transition"] , "expression": { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "defaults"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "provider": "NON_SYSTEM_TOOLS" , "transition": { "type": "var" , "name": "defaults-transition" , "default": {"type": "empty_map"} } , "default": {"type": "empty_map"} } } } } , "defaults-base-provides-list": { "doc": ["Query list of providers from 'base' targets"] , "vars": ["provider", "default"] , "vars_doc": { "provider": ["The name of the provider in the provides map."] , "default": ["The default if provider is missing (default: [])."] } , "imports": {"provider_list": ["./", "..", "field_provider_list"]} , "expression": { "type": "let*" , "bindings": [["fieldname", "base"]] , "body": {"type": "CALL_EXPRESSION", "name": "provider_list"} } } , "defaults-base-provides-++": { "doc": ["Query flattend list of providers from 'base' targets"] , "vars": ["provider", "default"] , "vars_doc": { "provider": ["The name of the provider in the provides map."] , "default": ["The default if provider is missing (default: [])."] } , "imports": {"base-provides-list": "defaults-base-provides-list"} , "expression": { "type": "++" , "$1": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} } } , "defaults-base-provides": { "doc": ["Query provider from 'base' targets (last wins)"] , "vars": ["provider", "default"] , "vars_doc": { "provider": ["The name of the provider in the provides map."] , "default": ["The default if provider is missing (default: [])."] } , "imports": {"base-provides-list": "defaults-base-provides-list"} , "expression": { "type": "foldl" , "var": "next" , "start": {"type": "var", "name": "default", "default": []} , "accum_var": "curr" , "range": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} , "body": { "type": "if" , "cond": {"type": "var", "name": "next"} , "then": {"type": "var", "name": "next"} , "else": {"type": "var", "name": "curr"} } } } , "debug-deps": { "doc": ["Collect debug dependencies (sources/headers) from given target_fields"] , "vars": ["deps-provider", "deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-provider": ["Name of provider to use (debug-srcs/debug-hdrs)."] , "deps-fieldnames": ["List of target_field names to collect dependencies from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"provider_list": ["./", "..", "field_provider_list"]} , "expression": { "type": "map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "++" , "$1": { "type": "let*" , "bindings": [ ["provider", {"type": "var", "name": "deps-provider"}] , ["transition", {"type": "var", "name": "deps-transition"}] , ["default", {"type": "empty_map"}] ] , "body": [{"type": "CALL_EXPRESSION", "name": "provider_list"}] } } } } } } , "compile-deps": { "doc": ["Collect compile dependencies (headers) from given target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect dependencies from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": { "runfiles_list": ["./", "..", "field_runfiles_list"] , "provider_list": ["./", "..", "field_provider_list"] } , "expression": { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "++" , "$1": { "type": "let*" , "bindings": [ ["provider", "compile-deps"] , ["transition", {"type": "var", "name": "deps-transition"}] , ["default", {"type": "empty_map"}] ] , "body": [ {"type": "CALL_EXPRESSION", "name": "provider_list"} , {"type": "CALL_EXPRESSION", "name": "runfiles_list"} ] } } } } } } , "compile-args-deps": { "doc": ["Collect compile arguments from given target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect arguments from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "nub_right" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "let*" , "bindings": [ ["provider", "compile-args"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } } } } , "link-deps": { "doc": ["Collect link dependencies (libraries) from given target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect dependencies from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": { "artifacts_list": ["./", "..", "field_artifacts_list"] , "provider_list": ["./", "..", "field_provider_list"] } , "expression": { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "++" , "$1": { "type": "let*" , "bindings": [ ["provider", "link-deps"] , ["default", {"type": "empty_map"}] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": [ {"type": "CALL_EXPRESSION", "name": "provider_list"} , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} ] } } } } } } , "link-args-deps": { "doc": ["Collect linker arguments from given target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect arguments from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "nub_right" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "let*" , "bindings": [ ["provider", "link-args"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } } } } , "pkg-map-provider-deps": { "doc": ["Collect maps from provider \"package\" for given target_fields"] , "vars": ["pkg-key", "deps-fieldnames", "deps-transition"] , "vars_doc": { "pkg-key": ["Key to lookup in provider \"package\"."] , "deps-fieldnames": ["List of target_field names to collect maps from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"provider_list": ["./", "..", "field_provider_list"]} , "expression": { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": { "type": "let*" , "bindings": [ ["provider", "package"] , ["default", {"type": "empty_map"}] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": { "type": "foreach" , "range": {"type": "CALL_EXPRESSION", "name": "provider_list"} , "var": "map" , "body": { "type": "lookup" , "key": {"type": "var", "name": "pkg-key"} , "map": {"type": "var", "name": "map"} , "default": {"type": "empty_map"} } } } } } } } , "cflags-files-deps": { "doc": ["Collect cflags files from target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect files from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"pkg-provider": "pkg-map-provider-deps"} , "expression": { "type": "let*" , "bindings": [["pkg-key", "cflags-files"]] , "body": {"type": "CALL_EXPRESSION", "name": "pkg-provider"} } } , "ldflags-files-deps": { "doc": ["Collect cflags files from target_fields"] , "vars": ["deps-fieldnames", "deps-transition"] , "vars_doc": { "deps-fieldnames": ["List of target_field names to collect files from."] , "deps-transition": ["The optional configuration transition for the targets."] } , "imports": {"pkg-provider": "pkg-map-provider-deps"} , "expression": { "type": "let*" , "bindings": [["pkg-key", "ldflags-files"]] , "body": {"type": "CALL_EXPRESSION", "name": "pkg-provider"} } } , "pkg-prefix-lib-paths": { "doc": ["Detect ldflags referring to local libs and prefix them."] , "vars": ["pkg-ldflags", "pkg-libs", "flat-libs", "lib-prefix"] , "expression": { "type": "let*" , "bindings": [ [ "libs set" , { "type": "set" , "$1": {"type": "keys", "$1": {"type": "var", "name": "pkg-libs"}} } ] ] , "body": { "type": "foreach" , "var": "item" , "range": {"type": "var", "name": "pkg-ldflags"} , "body": { "type": "if" , "cond": { "type": "lookup" , "key": {"type": "var", "name": "item"} , "map": {"type": "var", "name": "libs set"} } , "then": { "type": "join" , "$1": { "type": "if" , "cond": {"type": "var", "name": "flat-libs"} , "then": [ "-l:" , {"type": "basename", "$1": {"type": "var", "name": "item"}} ] , "else": [ {"type": "var", "name": "lib-prefix"} , "/" , {"type": "var", "name": "item"} ] } } , "else": {"type": "var", "name": "item"} } } } } , "pkg-prefix-flag-paths": { "doc": ["Detect flags referring to local flag files and prefix them."] , "vars": ["flags", "pkg-flag-files", "flag-prefix"] , "expression": { "type": "let*" , "bindings": [ [ "pkg-flag-files unprefix map" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "name" , "range": {"type": "keys", "$1": {"type": "var", "name": "pkg-flag-files"}} , "body": { "type": "singleton_map" , "key": {"type": "join", "$1": ["@", {"type": "var", "name": "name"}]} , "value": {"type": "var", "name": "name"} } } } ] ] , "body": { "type": "foreach" , "var": "item" , "range": {"type": "var", "name": "flags"} , "body": { "type": "let*" , "bindings": [ [ "flag-file" , { "type": "lookup" , "map": {"type": "var", "name": "pkg-flag-files unprefix map"} , "key": {"type": "var", "name": "item"} } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "flag-file"} , "then": { "type": "join" , "$1": [ "@" , {"type": "var", "name": "flag-prefix"} , "/" , {"type": "var", "name": "flag-file"} ] } , "else": {"type": "var", "name": "item"} } } } } } , "pkg-config": { "vars": [ "pkg-name" , "pkg-prefix" , "pkg-version" , "pkg-cflags" , "pkg-ldflags" , "pkg-flag-files" , "pkg-libs" , "flat-libs" , "pc-install-dir" ] , "imports": { "pkg-prefix-lib-paths": "pkg-prefix-lib-paths" , "pkg-prefix-flag-paths": "pkg-prefix-flag-paths" } , "expression": { "type": "let*" , "bindings": [ ["lib-prefix", "${libdir}"] , [ "pkg-ldflags" , {"type": "CALL_EXPRESSION", "name": "pkg-prefix-lib-paths"} ] , [ "flag-prefix" , { "type": "join" , "$1": ["${prefix}/", {"type": "var", "name": "pc-install-dir"}] } ] , ["flags", {"type": "var", "name": "pkg-cflags"}] , [ "pkg-cflags" , {"type": "CALL_EXPRESSION", "name": "pkg-prefix-flag-paths"} ] , ["flags", {"type": "var", "name": "pkg-ldflags"}] , [ "pkg-ldflags" , {"type": "CALL_EXPRESSION", "name": "pkg-prefix-flag-paths"} ] ] , "body": { "type": "singleton_map" , "key": {"type": "join", "$1": [{"type": "var", "name": "pkg-name"}, ".pc"]} , "value": { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ { "type": "join" , "$1": [ "prefix=" , {"type": "var", "name": "pkg-prefix", "default": "/"} ] } , "libdir=${prefix}/lib" , "includedir=${prefix}/include" , { "type": "join" , "$1": ["Name: ", {"type": "var", "name": "pkg-name"}] } , { "type": "join" , "$1": [ "Version: " , {"type": "var", "name": "pkg-version", "default": "unknown"} ] } , { "type": "join" , "$1": [ "Description: Pkg-config for " , {"type": "var", "name": "pkg-name"} , ", generated by JustBuild" ] } , "URL: unknown" , { "type": "join" , "separator": " " , "$1": { "type": "++" , "$1": [ ["Cflags:", "-I${includedir}"] , {"type": "var", "name": "pkg-cflags"} ] } } , { "type": "join" , "separator": " " , "$1": { "type": "++" , "$1": [ ["Libs:"] , { "type": "if" , "cond": {"type": "var", "name": "flat-libs"} , "then": ["-L${libdir}"] } , {"type": "var", "name": "pkg-ldflags"} ] } } , "" ] } } } } } , "add-fission-compile-flags": { "doc": ["Add debug fission flags, if enabled, to existing compile flags."] , "vars": ["COMPILE_FLAGS", "DEBUG"] , "expression": { "type": "let*" , "bindings": [ [ "fission-config" , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "then": { "type": "assert_non_empty" , "msg": [ "Debug fission requires non-empty debug map FISSION_CONFIG field" ] , "$1": { "type": "lookup" , "key": "FISSION_CONFIG" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } } , "else": {"type": "empty_map"} } ] , [ "flags" , { "type": "++" , "$1": [ {"type": "var", "name": "COMPILE_FLAGS"} , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_SPLIT_DWARF" , "map": {"type": "var", "name": "fission-config"} } , "then": ["-gsplit-dwarf"] } , { "type": "let*" , "bindings": [ [ "version" , { "type": "lookup" , "key": "DWARF_VERSION" , "map": {"type": "var", "name": "fission-config"} } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "version"} , "then": [ { "type": "join" , "$1": ["-gdwarf-", {"type": "var", "name": "version"}] } ] } } , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_TYPES_SECTION" , "map": {"type": "var", "name": "fission-config"} } , "then": ["-fdebug-types-section"] } ] } ] ] , "body": {"type": "var", "name": "flags"} } } , "add-fission-link-flags": { "doc": ["Add debug fission flags, if enabled, to existing link flags."] , "vars": ["LDFLAGS", "DEBUG"] , "expression": { "type": "let*" , "bindings": [ [ "fission-config" , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "then": { "type": "assert_non_empty" , "msg": [ "Debug fission requires non-empty debug map FISSION_CONFIG field" ] , "$1": { "type": "lookup" , "key": "FISSION_CONFIG" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } } , "else": {"type": "empty_map"} } ] , [ "flags" , { "type": "++" , "$1": [ {"type": "var", "name": "LDFLAGS"} , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_GDB_INDEX" , "map": {"type": "var", "name": "fission-config"} } , "then": ["-Wl,--gdb-index"] } ] } ] ] , "body": {"type": "var", "name": "flags"} } } , "dwarf package": { "vars": [ "DWP" , "TOOLCHAIN" , "TOOLCHAIN_DIR" , "NON_SYSTEM_TOOLS" , "ENV" , "name" , "dwarf objects" , "dwarf deps" , "stage" ] , "imports": {"default-DWP": "default-DWP"} , "expression": { "type": "let*" , "bindings": [ [ "DWP" , { "type": "var" , "name": "DWP" , "default": { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "DWP" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-DWP"} ] } } } ] , [ "pkgname" , {"type": "join", "$1": [{"type": "var", "name": "name"}, ".dwp"]} ] , [ "pkgpath" , { "type": "if" , "cond": {"type": "var", "name": "stage"} , "then": { "type": "join" , "separator": "/" , "$1": [ {"type": "var", "name": "stage"} , {"type": "var", "name": "pkgname"} ] } , "else": {"type": "var", "name": "pkgname"} } ] , [ "objects" , { "type": "var" , "name": "dwarf objects" , "default": {"type": "empty_map"} } ] , [ "objects" , { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "objects"} , {"type": "var", "name": "dwarf deps"} ] } ] , [ "dwarf pkg" , { "type": "if" , "cond": {"type": "var", "name": "objects"} , "else": {"type": "empty_map"} , "then": { "type": "let*" , "bindings": [ [ "staged objects" , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "objects"} } ] , [ "staged pkgpath" , { "type": "join" , "$1": ["work/", {"type": "var", "name": "pkgpath"}] } ] , [ "staged pkg" , { "type": "ACTION" , "outs": [{"type": "var", "name": "staged pkgpath"}] , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "var", "name": "staged objects"} ] } , "env": { "type": "var" , "name": "ENV" , "default": {"type": "empty_map"} } , "cmd": { "type": "++" , "$1": [ [ {"type": "var", "name": "DWP", "default": "dwp"} , "-o" , {"type": "var", "name": "staged pkgpath"} ] , { "type": "keys" , "$1": {"type": "var", "name": "staged objects"} } ] } } ] , [ "pkg artifact" , { "type": "lookup" , "map": {"type": "var", "name": "staged pkg"} , "key": {"type": "var", "name": "staged pkgpath"} } ] ] , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "pkgpath"} , "value": {"type": "var", "name": "pkg artifact"} } } } ] ] , "body": {"type": "var", "name": "dwarf pkg"} } } , "objects": { "vars": [ "COMPILER" , "COMPILE_FLAGS" , "ENV" , "DEBUG" , "TOOLCHAIN" , "srcs" , "hdrs" , "private-hdrs" , "compile-deps" , "cflags-files" ] , "imports": {"add-fission-compile-flags": "add-fission-compile-flags"} , "expression": { "type": "let*" , "bindings": [ [ "include tree" , { "type": "singleton_map" , "key": "include" , "value": {"type": "TREE", "$1": {"type": "var", "name": "compile-deps"}} } ] , [ "all hdrs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "include tree"} , { "type": "to_subdir" , "subdir": "work" , "$1": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "hdrs"} , {"type": "var", "name": "private-hdrs"} ] } } ] } ] , [ "compile flags" , {"type": "CALL_EXPRESSION", "name": "add-fission-compile-flags"} ] , [ "all staged output artifacts" , { "type": "foreach_map" , "var_key": "src_name" , "var_val": "src_val" , "range": {"type": "var", "name": "srcs"} , "body": { "type": "let*" , "bindings": [ [ "work src_name" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "src_name"}] } ] , [ "inputs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "all hdrs"} , { "type": "singleton_map" , "key": {"type": "var", "name": "work src_name"} , "value": {"type": "var", "name": "src_val"} } , { "type": "var" , "name": "cflags-files" , "default": {"type": "empty_map"} } , { "type": "var" , "name": "TOOLCHAIN" , "default": {"type": "empty_map"} } ] } ] , [ "out" , { "type": "change_ending" , "$1": {"type": "var", "name": "src_name"} , "ending": ".o" } ] , [ "work out" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "out"}] } ] , [ "dwarf out" , { "type": "change_ending" , "$1": {"type": "var", "name": "src_name"} , "ending": ".dwo" } ] , [ "work dwarf out" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "dwarf out"}] } ] , [ "action output" , { "type": "ACTION" , "outs": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "then": [{"type": "var", "name": "work dwarf out"}] , "else": [] } , [{"type": "var", "name": "work out"}] ] } , "inputs": {"type": "var", "name": "inputs"} , "env": { "type": "var" , "name": "ENV" , "default": {"type": "empty_map"} } , "cmd": { "type": "++" , "$1": [ [{"type": "var", "name": "COMPILER"}] , {"type": "var", "name": "compile flags"} , ["-I", "work", "-isystem", "include"] , ["-c", {"type": "var", "name": "work src_name"}] , ["-o", {"type": "var", "name": "work out"}] ] } } ] , [ "staged output artifacts" , { "type": "++" , "$1": [ [ { "type": "singleton_map" , "key": {"type": "var", "name": "out"} , "value": { "type": "lookup" , "key": {"type": "var", "name": "work out"} , "map": {"type": "var", "name": "action output"} } } ] , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "else": [] , "then": [ { "type": "singleton_map" , "key": {"type": "var", "name": "dwarf out"} , "value": { "type": "lookup" , "key": {"type": "var", "name": "work dwarf out"} , "map": {"type": "var", "name": "action output"} } } ] } ] } ] ] , "body": {"type": "var", "name": "staged output artifacts"} } } ] , [ "staged objects" , { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "all staged output artifacts"} , "body": { "type": "[]" , "index": "0" , "list": {"type": "var", "name": "_"} } } } ] , [ "staged dwarf objects" , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "else": {"type": "empty_map"} , "then": { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "all staged output artifacts"} , "body": { "type": "[]" , "index": "1" , "list": {"type": "var", "name": "_"} } } } } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "objects" , "value": {"type": "var", "name": "staged objects"} } , { "type": "singleton_map" , "key": "dwarf objects" , "value": {"type": "var", "name": "staged dwarf objects"} } ] } } } , "lint information": { "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "DEBUG" , "pure C" , "srcs" , "hdrs" , "private-hdrs" , "compile-deps" , "cflags-files" , "lint-deps fieldnames" , "deps-transition" , "deps-fieldnames" , "compile-args" , "defaults-transition" ] , "imports": { "list_provider": ["./", "..", "field_list_provider"] , "runfiles_list": ["./", "..", "field_runfiles_list"] , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" , "compiler": "compiler" , "flags": "flags" , "add-fission-compile-flags": "add-fission-compile-flags" } , "expression": { "type": "let*" , "bindings": [ ["TOOLCHAIN_DIR", "toolchain"] , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , ["COMPILER", {"type": "CALL_EXPRESSION", "name": "compiler"}] , [ "COMPILE_FLAGS" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "flags"} , {"type": "var", "name": "compile-args"} ] } ] , [ "compile flags" , {"type": "CALL_EXPRESSION", "name": "add-fission-compile-flags"} ] , [ "include tree" , { "type": "singleton_map" , "key": "include" , "value": {"type": "TREE", "$1": {"type": "var", "name": "compile-deps"}} } ] , [ "all hdrs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "include tree"} , { "type": "to_subdir" , "subdir": "work" , "$1": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "hdrs"} , {"type": "var", "name": "private-hdrs"} ] } } ] } ] , [ "direct-deps hdrs" , { "type": "to_subdir" , "subdir": "include" , "$1": { "type": "let*" , "bindings": [["transition", {"type": "var", "name": "deps-transition"}]] , "body": { "type": "map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "deps-fieldnames"} , "body": {"type": "CALL_EXPRESSION", "name": "runfiles_list"} } } } } } ] , [ "own headers" , { "type": "to_subdir" , "subdir": "work" , "$1": { "type": "map_union" , "$1": [ {"type": "var", "name": "hdrs"} , {"type": "var", "name": "private-hdrs"} ] } } ] , [ "direct hdrs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "own headers"} , {"type": "var", "name": "direct-deps hdrs"} ] } ] , [ "direct deps artifact names" , {"type": "keys", "$1": {"type": "var", "name": "direct hdrs"}} ] , [ "hdr lint" , { "type": "foreach" , "range": { "type": "++" , "$1": [ {"type": "keys", "$1": {"type": "var", "name": "private-hdrs"}} , {"type": "keys", "$1": {"type": "var", "name": "hdrs"}} ] } , "body": { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": {"type": "var", "name": "all hdrs"} , "provides": { "type": "let*" , "bindings": [ [ "src" , { "type": "join" , "$1": ["work/", {"type": "var", "name": "_"}] } ] , [ "cmd" , { "type": "++" , "$1": [ [{"type": "var", "name": "COMPILER"}] , {"type": "var", "name": "compile flags"} , [ "-I" , "work" , "-isystem" , "include" , "-E" , {"type": "var", "name": "src"} ] ] } ] , ["extra outs", []] ] , "body": { "type": "env" , "vars": ["cmd", "src", "direct deps artifact names", "extra outs"] } } } } } ] , [ "src lint" , { "type": "foreach_map" , "var_key": "src_name" , "var_val": "src_val" , "range": {"type": "var", "name": "srcs"} , "body": { "type": "let*" , "bindings": [ [ "work src_name" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "src_name"}] } ] , [ "inputs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "all hdrs"} , { "type": "singleton_map" , "key": {"type": "var", "name": "work src_name"} , "value": {"type": "var", "name": "src_val"} } , { "type": "var" , "name": "cflags-files" , "default": {"type": "empty_map"} } , { "type": "var" , "name": "TOOLCHAIN" , "default": {"type": "empty_map"} } ] } ] , [ "out" , { "type": "change_ending" , "$1": {"type": "var", "name": "src_name"} , "ending": ".o" } ] , [ "work out" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "out"}] } ] , [ "dwarf out" , { "type": "change_ending" , "$1": {"type": "var", "name": "src_name"} , "ending": ".dwo" } ] , [ "work dwarf out" , { "type": "join" , "separator": "/" , "$1": ["work", {"type": "var", "name": "dwarf out"}] } ] , [ "extra outs" , { "type": "if" , "cond": { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } , "then": [{"type": "var", "name": "work dwarf out"}] , "else": [] } ] , [ "cmd" , { "type": "++" , "$1": [ [{"type": "var", "name": "COMPILER"}] , {"type": "var", "name": "compile flags"} , ["-I", "work", "-isystem", "include"] , ["-c", {"type": "var", "name": "work src_name"}] , ["-o", {"type": "var", "name": "work out"}] ] } ] ] , "body": { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": {"type": "var", "name": "inputs"} , "provides": { "type": "let*" , "bindings": [["src", {"type": "var", "name": "work src_name"}]] , "body": { "type": "env" , "vars": [ "cmd" , "src" , "direct deps artifact names" , "extra outs" ] } } } } } } ] , [ "dep lint nodes" , { "type": "++" , "$1": { "type": "foreach" , "var": "fieldname" , "range": {"type": "var", "name": "lint-deps fieldnames"} , "body": { "type": "let*" , "bindings": [ ["provider", "lint"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } } ] , [ "lint nodes" , { "type": "++" , "$1": [ {"type": "var", "name": "hdr lint"} , {"type": "var", "name": "src lint"} , {"type": "var", "name": "dep lint nodes"} ] } ] , [ "lint nodes" , {"type": "nub_right", "$1": {"type": "var", "name": "lint nodes"}} ] ] , "body": {"type": "var", "name": "lint nodes"} } } , "compiler-cc": { "vars": ["CC", "TOOLCHAIN_DIR", "NON_SYSTEM_TOOLS", "defaults-transition"] , "imports": {"default-CC": "default-CC"} , "expression": { "type": "var" , "name": "CC" , "default": { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "CC" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-CC"} ] } } } } , "compiler-cxx": { "vars": ["CXX", "TOOLCHAIN_DIR", "NON_SYSTEM_TOOLS", "defaults-transition"] , "imports": {"default-CXX": "default-CXX"} , "expression": { "type": "var" , "name": "CXX" , "default": { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "CXX" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-CXX"} ] } } } } , "compiler": { "vars": [ "CC" , "CXX" , "pure C" , "TOOLCHAIN_DIR" , "NON_SYSTEM_TOOLS" , "defaults-transition" ] , "imports": {"compiler-cc": "compiler-cc", "compiler-cxx": "compiler-cxx"} , "expression": { "type": "if" , "cond": {"type": "var", "name": "pure C"} , "then": {"type": "CALL_EXPRESSION", "name": "compiler-cc"} , "else": {"type": "CALL_EXPRESSION", "name": "compiler-cxx"} } } , "flags-cc": { "vars": ["CFLAGS", "ADD_CFLAGS", "defaults-transition"] , "imports": {"default-CFLAGS": "default-CFLAGS"} , "expression": { "type": "++" , "$1": [ { "type": "var" , "name": "CFLAGS" , "default": {"type": "CALL_EXPRESSION", "name": "default-CFLAGS"} } , {"type": "var", "name": "ADD_CFLAGS", "default": []} ] } } , "flags-cxx": { "vars": ["CXXFLAGS", "ADD_CXXFLAGS", "defaults-transition"] , "imports": {"default-CXXFLAGS": "default-CXXFLAGS"} , "expression": { "type": "++" , "$1": [ { "type": "var" , "name": "CXXFLAGS" , "default": {"type": "CALL_EXPRESSION", "name": "default-CXXFLAGS"} } , {"type": "var", "name": "ADD_CXXFLAGS", "default": []} ] } } , "flags": { "vars": [ "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "DEBUG" , "pure C" , "defaults-transition" ] , "imports": {"flags-cc": "flags-cc", "flags-cxx": "flags-cxx"} , "expression": { "type": "let*" , "bindings": [ [ "flags" , { "type": "if" , "cond": {"type": "var", "name": "pure C"} , "then": {"type": "CALL_EXPRESSION", "name": "flags-cc"} , "else": {"type": "CALL_EXPRESSION", "name": "flags-cxx"} } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "flags"} , "then": {"type": "var", "name": "flags"} , "else": { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": ["-g"] } } } } , "lib artifact": { "doc": ["Provides the library artifact."] , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "DEBUG" , "name" , "pure C" , "srcs" , "hdrs" , "private-hdrs" , "stage" , "compile-deps" , "compile-args" , "cflags-files" , "defaults-transition" ] , "imports": {"lib action": "lib action"} , "expression": { "type": "lookup" , "key": "library" , "map": {"type": "CALL_EXPRESSION", "name": "lib action"} , "default": {"type": "empty_map"} } } , "lib action": { "doc": [ "Run the action producing the library artifact and pass it together with" , "related information to consumers." ] , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "DEBUG" , "name" , "pure C" , "srcs" , "hdrs" , "private-hdrs" , "stage" , "compile-deps" , "compile-args" , "cflags-files" , "defaults-transition" ] , "imports": { "compiler": "compiler" , "flags": "flags" , "objects": "objects" , "default-AR": "default-AR" , "default-ARFLAGS": "default-ARFLAGS" , "default-ENV": "default-ENV" , "default-PATH": "default-PATH" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" } , "expression": { "type": "let*" , "bindings": [ ["TOOLCHAIN_DIR", "toolchain"] , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , ["COMPILER", {"type": "CALL_EXPRESSION", "name": "compiler"}] , [ "COMPILE_FLAGS" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "flags"} , {"type": "var", "name": "compile-args"} ] } ] , [ "AR" , { "type": "var" , "name": "AR" , "default": { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "AR" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-AR"} ] } } } ] , ["ARFLAGS", {"type": "CALL_EXPRESSION", "name": "default-ARFLAGS"}] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , ["objects result", {"type": "CALL_EXPRESSION", "name": "objects"}] , [ "objects" , { "type": "lookup" , "key": "objects" , "map": { "type": "var" , "name": "objects result" , "default": {"type": "empty_map"} } } ] , [ "base name" , { "type": "if" , "cond": {"type": "var", "name": "objects"} , "then": { "type": "assert_non_empty" , "msg": "A name has to be provided for non-header-only libraries" , "$1": {"type": "var", "name": "name"} } , "else": {"type": "var", "name": "name"} } ] , [ "libname" , { "type": "join" , "$1": ["lib", {"type": "var", "name": "base name"}, ".a"] } ] , [ "libpath" , { "type": "if" , "cond": {"type": "var", "name": "stage"} , "then": { "type": "join" , "separator": "/" , "$1": [ {"type": "var", "name": "stage"} , {"type": "var", "name": "libname"} ] } , "else": {"type": "var", "name": "libname"} } ] , [ "lib" , { "type": "if" , "cond": {"type": "var", "name": "objects"} , "else": {"type": "empty_map"} , "then": { "type": "let*" , "bindings": [ [ "staged objects" , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "objects"} } ] , [ "staged libpath" , { "type": "join" , "$1": ["work/", {"type": "var", "name": "libpath"}] } ] , [ "staged lib" , { "type": "ACTION" , "outs": [{"type": "var", "name": "staged libpath"}] , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "var", "name": "staged objects"} ] } , "env": { "type": "var" , "name": "ENV" , "default": {"type": "empty_map"} } , "cmd": { "type": "++" , "$1": [ [{"type": "var", "name": "AR", "default": "ar"}] , {"type": "var", "name": "ARFLAGS"} , [{"type": "var", "name": "staged libpath"}] , { "type": "keys" , "$1": {"type": "var", "name": "staged objects"} } ] } } ] , [ "lib artifact" , { "type": "lookup" , "map": {"type": "var", "name": "staged lib"} , "key": {"type": "var", "name": "staged libpath"} } ] ] , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "libpath"} , "value": {"type": "var", "name": "lib artifact"} } } } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "library" , "value": {"type": "var", "name": "lib"} } , { "type": "singleton_map" , "key": "dwarf objects" , "value": { "type": "lookup" , "key": "dwarf objects" , "map": { "type": "var" , "name": "objects result" , "default": {"type": "empty_map"} } } } , { "type": "env" , "vars": ["TOOLCHAIN", "TOOLCHAIN_DIR", "NON_SYSTEM_TOOLS", "ENV"] } ] } } } , "lib result": { "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "DWP" , "ENV" , "DEBUG" , "LINT" , "name" , "pure C" , "srcs" , "hdrs" , "private-hdrs" , "cflags" , "private-cflags" , "private-ldflags" , "stage" , "pkg-name" , "extra-provides" , "public-fieldnames" , "private-fieldnames" ] , "imports": { "compile-deps": "compile-deps" , "compile-args-deps": "compile-args-deps" , "link-deps": "link-deps" , "link-args-deps": "link-args-deps" , "cflags-files-deps": "cflags-files-deps" , "ldflags-files-deps": "ldflags-files-deps" , "lib action": "lib action" , "debug-deps": "debug-deps" , "lint": "lint information" , "dwarf package": "dwarf package" } , "expression": { "type": "let*" , "bindings": [ ["deps-fieldnames", {"type": "var", "name": "private-fieldnames"}] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "cflags"} , {"type": "var", "name": "private-cflags"} , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] } } ] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , [ "lib action result" , {"type": "CALL_EXPRESSION", "name": "lib action"} ] , [ "lib" , { "type": "lookup" , "key": "library" , "map": {"type": "var", "name": "lib action result"} } ] , [ "dwarf objects" , { "type": "lookup" , "key": "dwarf objects" , "map": {"type": "var", "name": "lib action result"} } ] , ["dwarf deps", {"type": "empty_map"}] , [ "dwarf-pkg" , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "dwarf objects"} , { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } ] } , "then": { "type": "let*" , "bindings": [ [ "TOOLCHAIN" , { "type": "lookup" , "key": "TOOLCHAIN" , "map": {"type": "var", "name": "lib action result"} } ] , [ "TOOLCHAIN_DIR" , { "type": "lookup" , "key": "TOOLCHAIN_DIR" , "map": {"type": "var", "name": "lib action result"} } ] , [ "NON_SYSTEM_TOOLS" , { "type": "lookup" , "key": "NON_SYSTEM_TOOLS" , "map": {"type": "var", "name": "lib action result"} } ] , [ "ENV" , { "type": "lookup" , "key": "ENV" , "map": {"type": "var", "name": "lib action result"} } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "dwarf package"} } , "else": {"type": "empty_map"} } ] , ["lint-deps fieldnames", ["deps", "private-deps"]] , [ "lint" , { "type": "if" , "cond": {"type": "var", "name": "LINT"} , "then": {"type": "CALL_EXPRESSION", "name": "lint"} } ] , [ "link-args" , { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "keys", "$1": {"type": "var", "name": "lib"}} , {"type": "var", "name": "private-ldflags", "default": []} , {"type": "CALL_EXPRESSION", "name": "link-args-deps"} ] } } ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , [ "debug-srcs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "srcs"} , {"type": "var", "name": "private-hdrs"} , {"type": "var", "name": "hdrs"} ] } , "else": {"type": "empty_map"} } ] , [ "debug-hdrs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "hdrs"} ] } , "else": {"type": "empty_map"} } ] , ["deps-fieldnames", {"type": "var", "name": "public-fieldnames"}] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "cflags"} , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] } } ] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , [ "package" , { "type": "let*" , "bindings": [["name", {"type": "var", "name": "pkg-name"}]] , "body": {"type": "env", "vars": ["name", "cflags-files", "ldflags-files"]} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "lib"} , "runfiles": {"type": "var", "name": "hdrs"} , "provides": { "type": "map_union" , "$1": [ { "type": "env" , "vars": [ "compile-deps" , "compile-args" , "link-deps" , "link-args" , "package" , "debug-srcs" , "debug-hdrs" , "lint" , "dwarf-pkg" ] } , { "type": "var" , "name": "extra-provides" , "default": {"type": "empty_map"} } ] } } } } , "bin artifact": { "doc": ["Provides the linked binary."] , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "DEBUG" , "name" , "pure C" , "srcs" , "private-hdrs" , "stage" , "compile-deps" , "compile-args" , "link-deps" , "link-args" , "cflags-files" , "ldflags-files" , "defaults-transition" ] , "imports": {"bin action": "bin action"} , "expression": { "type": "lookup" , "key": "binary" , "map": {"type": "CALL_EXPRESSION", "name": "bin action"} , "default": {"type": "empty_map"} } } , "bin action": { "doc": [ "Run the action producing the binary artifact and pass it together with" , "related information to consumers." ] , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "DEBUG" , "name" , "pure C" , "srcs" , "private-hdrs" , "stage" , "compile-deps" , "compile-args" , "link-deps" , "link-args" , "cflags-files" , "ldflags-files" , "defaults-transition" ] , "imports": { "compiler": "compiler" , "flags": "flags" , "objects": "objects" , "default-ENV": "default-ENV" , "default-PATH": "default-PATH" , "default-LDFLAGS": "default-LDFLAGS" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" , "add-fission-compile-flags": "add-fission-compile-flags" , "add-fission-link-flags": "add-fission-link-flags" } , "expression": { "type": "let*" , "bindings": [ ["TOOLCHAIN_DIR", "toolchain"] , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , ["COMPILER", {"type": "CALL_EXPRESSION", "name": "compiler"}] , [ "COMPILE_FLAGS" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "flags"} , {"type": "var", "name": "compile-args"} ] } ] , ["LDFLAGS", {"type": "CALL_EXPRESSION", "name": "default-LDFLAGS"}] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , ["hdrs", {"type": "empty_map"}] , ["objects result", {"type": "CALL_EXPRESSION", "name": "objects"}] , [ "objects" , { "type": "lookup" , "key": "objects" , "map": { "type": "var" , "name": "objects result" , "default": {"type": "empty_map"} } } ] , [ "compile flags" , {"type": "CALL_EXPRESSION", "name": "add-fission-compile-flags"} ] , [ "ld flags" , {"type": "CALL_EXPRESSION", "name": "add-fission-link-flags"} ] , [ "link-args" , { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "keys", "$1": {"type": "var", "name": "objects"}} , {"type": "var", "name": "link-args"} ] } } ] , [ "binpath" , { "type": "if" , "cond": {"type": "var", "name": "stage"} , "then": { "type": "join" , "separator": "/" , "$1": [ {"type": "var", "name": "stage"} , {"type": "var", "name": "name"} ] } , "else": {"type": "var", "name": "name"} } ] , [ "binpath (in work)" , {"type": "join", "$1": ["work/", {"type": "var", "name": "binpath"}]} ] , ["TOOLCHAIN_DIR", "../toolchain"] , ["COMPILER", {"type": "CALL_EXPRESSION", "name": "compiler"}] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , [ "cmd" , { "type": "++" , "$1": [ [ {"type": "var", "name": "COMPILER"} , "-o" , {"type": "var", "name": "binpath"} ] , {"type": "var", "name": "compile flags"} , {"type": "var", "name": "link-args"} , {"type": "var", "name": "ld flags"} ] } ] , [ "binary (in work)" , { "type": "ACTION" , "outs": [{"type": "var", "name": "binpath (in work)"}] , "inputs": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , { "type": "to_subdir" , "subdir": "work" , "$1": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "objects"} , {"type": "var", "name": "link-deps"} , { "type": "var" , "name": "cflags-files" , "default": {"type": "empty_map"} } , { "type": "var" , "name": "ldflags-files" , "default": {"type": "empty_map"} } ] } } ] } , "cmd": {"type": "var", "name": "cmd"} , "cwd": "work" , "env": {"type": "var", "name": "ENV"} } ] , [ "binary" , { "type": "singleton_map" , "key": {"type": "var", "name": "binpath"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "binary (in work)"} , "key": {"type": "var", "name": "binpath (in work)"} } } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "binary" , "value": {"type": "var", "name": "binary"} } , { "type": "singleton_map" , "key": "dwarf objects" , "value": { "type": "lookup" , "key": "dwarf objects" , "map": { "type": "var" , "name": "objects result" , "default": {"type": "empty_map"} } } } , { "type": "env" , "vars": ["TOOLCHAIN", "TOOLCHAIN_DIR", "NON_SYSTEM_TOOLS", "ENV"] } ] } } } , "bin result": { "doc": ["Produces the binary target result."] , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "DEBUG" , "LINT" , "name" , "pure C" , "srcs" , "private-hdrs" , "private-cflags" , "private-ldflags" , "stage" , "private-fieldnames" ] , "imports": { "compile-deps": "compile-deps" , "compile-args-deps": "compile-args-deps" , "link-deps": "link-deps" , "link-args-deps": "link-args-deps" , "cflags-files-deps": "cflags-files-deps" , "ldflags-files-deps": "ldflags-files-deps" , "bin action": "bin action" , "debug-deps": "debug-deps" , "lint": "lint information" , "dwarf package": "dwarf package" } , "expression": { "type": "let*" , "bindings": [ ["deps-fieldnames", {"type": "var", "name": "private-fieldnames"}] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} , {"type": "var", "name": "private-cflags"} ] } ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , [ "link-args" , { "type": "nub_right" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "private-ldflags"} , {"type": "CALL_EXPRESSION", "name": "link-args-deps"} ] } } ] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , ["package", {"type": "singleton_map", "key": "to_bin", "value": true}] , [ "debug-srcs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "srcs"} , {"type": "var", "name": "private-hdrs"} ] } , "else": {"type": "empty_map"} } ] , [ "debug-hdrs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "compile-deps"} ] } , "else": {"type": "empty_map"} } ] , [ "lint" , { "type": "if" , "cond": {"type": "var", "name": "LINT"} , "then": { "type": "let*" , "bindings": [ ["hdrs", {"type": "empty_map"}] , ["lint-deps fieldnames", ["private-deps"]] ] , "body": {"type": "CALL_EXPRESSION", "name": "lint"} } } ] , [ "bin action result" , {"type": "CALL_EXPRESSION", "name": "bin action"} ] , [ "binary" , { "type": "lookup" , "key": "binary" , "map": {"type": "var", "name": "bin action result"} } ] , [ "dwarf objects" , { "type": "lookup" , "key": "dwarf objects" , "map": {"type": "var", "name": "bin action result"} } ] , [ "dwarf deps" , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "dwarf objects"} , { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } ] } , "else": {"type": "empty_map"} , "then": { "type": "let*" , "bindings": [["deps-provider", "dwarf-pkg"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } } ] , [ "dwarf-pkg" , { "type": "if" , "cond": { "type": "and" , "$1": [ {"type": "var", "name": "dwarf objects"} , { "type": "lookup" , "key": "USE_DEBUG_FISSION" , "map": { "type": "var" , "name": "DEBUG" , "default": {"type": "empty_map"} } } ] } , "then": { "type": "let*" , "bindings": [ [ "TOOLCHAIN" , { "type": "lookup" , "key": "TOOLCHAIN" , "map": {"type": "var", "name": "bin action result"} } ] , [ "TOOLCHAIN_DIR" , { "type": "lookup" , "key": "TOOLCHAIN_DIR" , "map": {"type": "var", "name": "bin action result"} } ] , [ "NON_SYSTEM_TOOLS" , { "type": "lookup" , "key": "NON_SYSTEM_TOOLS" , "map": {"type": "var", "name": "bin action result"} } ] , [ "ENV" , { "type": "lookup" , "key": "ENV" , "map": {"type": "var", "name": "bin action result"} } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "dwarf package"} } , "else": {"type": "empty_map"} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "binary"} , "provides": { "type": "env" , "vars": ["package", "debug-srcs", "debug-hdrs", "lint", "dwarf-pkg"] } } } } , "install-with-deps result": { "vars": [ "pc-install-dir" , "targets" , "prefix" , "flat-libs" , "hdrs-only" , "skip-debug-stage" ] , "imports": {"pkg-config": "pkg-config"} , "expression": { "type": "let*" , "bindings": [ [ "install-stage" , { "type": "disjoint_map_union" , "msg": "install stages may not overlap" , "$1": { "type": "foreach" , "var": "target" , "range": {"type": "var", "name": "targets"} , "body": { "type": "let*" , "bindings": [ [ "runfiles" , { "type": "DEP_RUNFILES" , "dep": {"type": "var", "name": "target"} , "default": {"type": "empty_map"} } ] , [ "compile-deps" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "compile-deps" , "default": {"type": "empty_map"} } ] , [ "headers" , { "type": "disjoint_map_union" , "msg": "headers may not overlap" , "$1": [ {"type": "var", "name": "runfiles"} , {"type": "var", "name": "compile-deps"} ] } ] , [ "debug-srcs" , { "type": "if" , "cond": {"type": "var", "name": "skip-debug-stage"} , "then": {"type": "empty_map"} , "else": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "debug-srcs" , "default": {"type": "empty_map"} } } ] , [ "debug-hdrs" , { "type": "if" , "cond": {"type": "var", "name": "skip-debug-stage"} , "then": {"type": "empty_map"} , "else": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "debug-hdrs" , "default": {"type": "empty_map"} } } ] , [ "dwarf-pkg" , { "type": "if" , "cond": {"type": "var", "name": "skip-debug-stage"} , "then": {"type": "empty_map"} , "else": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "dwarf-pkg" , "default": {"type": "empty_map"} } } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "hdrs-only"} , "then": {"type": "var", "name": "headers"} , "else": { "type": "let*" , "bindings": [ [ "artifacts" , { "type": "map_union" , "$1": [ { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "target"} , "default": {"type": "empty_map"} } , {"type": "var", "name": "dwarf-pkg"} ] } ] , [ "link-deps" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "link-deps" , "default": {"type": "empty_map"} } ] , [ "package" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "package" , "default": {"type": "empty_map"} } ] , [ "to_bin" , { "type": "lookup" , "key": "to_bin" , "map": {"type": "var", "name": "package"} } ] , [ "binaries" , { "type": "if" , "cond": {"type": "var", "name": "to_bin"} , "then": {"type": "var", "name": "artifacts"} , "else": {"type": "empty_map"} } ] , [ "libraries" , { "type": "disjoint_map_union" , "msg": "libraries may not overlap" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "to_bin"} , "then": [] , "else": [{"type": "var", "name": "artifacts"}] } , [{"type": "var", "name": "link-deps"}] ] } } ] , [ "pkg-name" , { "type": "lookup" , "key": "name" , "map": {"type": "var", "name": "package"} } ] , [ "pkg-config" , { "type": "if" , "cond": {"type": "var", "name": "pkg-name"} , "then": { "type": "let*" , "bindings": [ ["pkg-prefix", {"type": "var", "name": "prefix"}] , [ "pkg-version" , { "type": "lookup" , "key": "version" , "map": {"type": "var", "name": "package"} } ] , [ "pkg-cflags" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "compile-args" , "default": [] } ] , [ "pkg-ldflags" , { "type": "++" , "$1": [ { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "target"} , "provider": "link-args" , "default": [] } ] } ] , [ "pkg-flag-files" , { "type": "map_union" , "$1": [ { "type": "lookup" , "key": "cflags-files" , "map": {"type": "var", "name": "package"} , "default": {"type": "empty_map"} } , { "type": "lookup" , "key": "ldflags-files" , "map": {"type": "var", "name": "package"} , "default": {"type": "empty_map"} } ] } ] , [ "pkg-libs" , {"type": "var", "name": "libraries"} ] , [ "flat-libs" , {"type": "var", "name": "flat-libs"} ] ] , "body": { "type": "map_union" , "$1": [ { "type": "CALL_EXPRESSION" , "name": "pkg-config" } , {"type": "var", "name": "pkg-flag-files"} ] } } , "else": {"type": "empty_map"} } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "to_subdir" , "subdir": "bin" , "flat": true , "msg": "install binaries may not overlap" , "$1": {"type": "var", "name": "binaries"} } , { "type": "to_subdir" , "subdir": "include" , "$1": { "type": "map_union" , "$1": [ {"type": "var", "name": "headers"} , {"type": "var", "name": "debug-hdrs"} ] } } , { "type": "to_subdir" , "subdir": "lib" , "flat": {"type": "var", "name": "flat-libs"} , "msg": "install libraries may not overlap" , "$1": {"type": "var", "name": "libraries"} } , { "type": "to_subdir" , "subdir": {"type": "var", "name": "pc-install-dir"} , "$1": {"type": "var", "name": "pkg-config"} } , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "debug-srcs"} } ] } } } } } } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "install-stage"}} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/RULES000066400000000000000000001057301516554100600220610ustar00rootroot00000000000000{ "defaults": { "doc": [ "A rule to provide defaults." , "All CC targets take their defaults for CC, CXX, flags, etc from" , "the target [\"CC\", \"defaults\"]. This is probably the only sensible" , "use of this rule. As targets form a different root, the defaults" , "can be provided without changing this directory." ] , "target_fields": ["base", "toolchain", "deps"] , "string_fields": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "LDFLAGS" , "ARFLAGS" , "DEBUGFLAGS" , "ADD_COMPILE_FLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ADD_LDFLAGS" , "ADD_DEBUGFLAGS" , "AR" , "DWP" , "PATH" , "SYSTEM_TOOLS" ] , "config_vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH", "DEBUG"] , "field_doc": { "base": ["Other targets (using the same rule) to inherit values from."] , "toolchain": [ "Optional toolchain directory. A collection of artifacts that provide" , "the tools CC, CXX, and AR (if needed). Note that only artifacts of" , "the specified targets are considered (no runfiles etc.). Specifying" , "this field extends artifacts from \"base\". If the toolchain" , "supports cross-compilation, it should perform a dispatch on the" , "configuration variable \"BUILD_ARCH\" to determine for which" , "architecture to generate code for." ] , "deps": [ "Optional CC libraries any CC library and CC binary implicitly depend" , "on. Those are typically \"libstdc++\" or \"libc++\" for C++ targets." , "Specifying this field extends dependencies from \"base\"." ] , "CC": ["The C compiler to use"] , "CXX": ["The C++ compiler to use"] , "AR": ["The archiver tool to use"] , "DWP": [ "The DWARF format packaging tool to use. Required by debug builds that" , "enable debug fission." ] , "SYSTEM_TOOLS": [ "List of tools (\"CC\", \"CXX\", \"AR\", or \"DWP\") that should be" , "taken from the system instead of from \"toolchain\" (if specified)." ] , "CFLAGS": [ "Flags for C compilation. Specifying this field overwrites" , "values from \"base\"." ] , "CXXFLAGS": [ "Flags for C++ compilation. Specifying this field overwrites" , "values from \"base\"." ] , "LDFLAGS": [ "Linker flags for linking the final CC library. Specifying this field" , "overwrites values from \"base\"." ] , "ARFLAGS": [ "Arguments to tell the archiver to create an archive with the specified" , "object files. If the \"ARFLAGS\" specified in the defaults target are" , "empty, the rules will use [\"cqs\"]." ] , "DEBUGFLAGS": [ "Flags to be used for the debug-stage for both C and C++, instead of" , "the resulting CFLAGS and CXXFLAGS, respectively." , "Specifying this field overwrites values from \"base\"." ] , "ADD_COMPILE_FLAGS": [ "Additional compilation flags for C and C++. Specifying this field" , "extends values from \"base\" for both, \"CFLAGS\" and \"CXXFLAGS\"." ] , "ADD_CFLAGS": [ "Additional compilation flags specific for C. Specifying this field" , "extends values from \"base\"." ] , "ADD_CXXFLAGS": [ "Additional compilation flags specific for C++. Specifying this field" , "extends values from \"base\"." ] , "ADD_LDFLAGS": [ "Additional linker flags for linking the final CC library. Specifying" , "this field extends values from \"base\"." ] , "ADD_DEBUGFLAGS": [ "Additional compilation flags for the debug-stage. Specifying this" , "field extends values from \"base\" for both, \"CFLAGS\" and" , "\"CXXFLAGS\"." ] , "PATH": [ "Path for looking up the compilers. Individual paths are joined" , "with \":\". Specifying this field extends values from \"base\"." ] } , "config_doc": { "ARCH": [ "The unqualified architecture. Is taken as a default for \"HOST_ARCH\"" , "and \"TARGET_ARCH\" if not set." ] , "HOST_ARCH": ["The architecture on which the build actions are carried out."] , "TARGET_ARCH": ["The architecture for which to build."] , "DEBUG": [ "If logically true (typically, a non-empty map), use debug-related" , "options, otherwise not." ] } , "imports": { "base-provides": "defaults-base-provides" , "base-provides-++": "defaults-base-provides-++" , "base-provides-list": "defaults-base-provides-list" , "artifacts_list": ["./", "..", "field_artifacts_list"] , "compile-deps": "compile-deps" , "compile-args-deps": "compile-args-deps" , "link-deps": "link-deps" , "link-args-deps": "link-args-deps" , "cflags-files-deps": "cflags-files-deps" , "ldflags-files-deps": "ldflags-files-deps" , "for host": ["transitions", "for host"] , "debug-deps": "debug-deps" } , "config_transitions": {"toolchain": [{"type": "CALL_EXPRESSION", "name": "for host"}]} , "expression": { "type": "let*" , "bindings": [ ["CC", {"type": "FIELD", "name": "CC"}] , ["CXX", {"type": "FIELD", "name": "CXX"}] , ["CFLAGS", {"type": "FIELD", "name": "CFLAGS"}] , ["CXXFLAGS", {"type": "FIELD", "name": "CXXFLAGS"}] , ["LDFLAGS", {"type": "FIELD", "name": "LDFLAGS"}] , ["ARFLAGS", {"type": "FIELD", "name": "ARFLAGS"}] , ["DEBUGFLAGS", {"type": "FIELD", "name": "DEBUGFLAGS"}] , ["AR", {"type": "FIELD", "name": "AR"}] , ["DWP", {"type": "FIELD", "name": "DWP"}] , ["PATH", {"type": "FIELD", "name": "PATH"}] , ["provider", "CC"] , [ "CC" , { "type": "if" , "cond": {"type": "var", "name": "CC"} , "then": {"type": "var", "name": "CC"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "CXX"] , [ "CXX" , { "type": "if" , "cond": {"type": "var", "name": "CXX"} , "then": {"type": "var", "name": "CXX"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "CFLAGS"] , [ "CFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "CFLAGS"} , "then": {"type": "var", "name": "CFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "CXXFLAGS"] , [ "CXXFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "CXXFLAGS"} , "then": {"type": "var", "name": "CXXFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "LDFLAGS"] , [ "LDFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "LDFLAGS"} , "then": {"type": "var", "name": "LDFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "ARFLAGS"] , [ "ARFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "ARFLAGS"} , "then": {"type": "var", "name": "ARFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "DEBUGFLAGS"] , [ "DEBUGFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "DEBUGFLAGS"} , "then": {"type": "var", "name": "DEBUGFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "AR"] , [ "AR" , { "type": "if" , "cond": {"type": "var", "name": "AR"} , "then": {"type": "var", "name": "AR"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "DWP"] , [ "DWP" , { "type": "if" , "cond": {"type": "var", "name": "DWP"} , "then": {"type": "var", "name": "DWP"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "PATH"] , [ "PATH" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "PATH"} , {"type": "CALL_EXPRESSION", "name": "base-provides-++"} ] } } ] , ["provider", "ENV"] , ["default", {"type": "empty_map"}] , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] , ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] , [ "NON_SYSTEM_TOOLS" , { "type": "map_union" , "$1": { "type": "++" , "$1": [ [{"type": "CALL_EXPRESSION", "name": "base-provides"}] , { "type": "if" , "cond": {"type": "FIELD", "name": "CC"} , "then": [ { "type": "singleton_map" , "key": "CC" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "if" , "cond": {"type": "FIELD", "name": "CXX"} , "then": [ { "type": "singleton_map" , "key": "CXX" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "if" , "cond": {"type": "FIELD", "name": "AR"} , "then": [ { "type": "singleton_map" , "key": "AR" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "if" , "cond": {"type": "FIELD", "name": "DWP"} , "then": [ { "type": "singleton_map" , "key": "DWP" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "foreach" , "range": {"type": "FIELD", "name": "SYSTEM_TOOLS"} , "var": "tool" , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "tool"} , "value": false } } ] } } ] , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] , [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "toolchain artifacts may not overlap" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} , { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": { "type": "let*" , "bindings": [ ["fieldname", "toolchain"] , [ "transition" , {"type": "CALL_EXPRESSION", "name": "for host"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } ] } } ] , [ "CFLAGS" , { "type": "++" , "$1": [ {"type": "var", "name": "CFLAGS"} , {"type": "FIELD", "name": "ADD_COMPILE_FLAGS"} , {"type": "FIELD", "name": "ADD_CFLAGS"} ] } ] , [ "CXXFLAGS" , { "type": "++" , "$1": [ {"type": "var", "name": "CXXFLAGS"} , {"type": "FIELD", "name": "ADD_COMPILE_FLAGS"} , {"type": "FIELD", "name": "ADD_CXXFLAGS"} ] } ] , [ "LDFLAGS" , { "type": "++" , "$1": [ {"type": "var", "name": "LDFLAGS"} , {"type": "FIELD", "name": "ADD_LDFLAGS"} ] } ] , [ "CFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "DEBUGFLAGS"} , "then": {"type": "var", "name": "DEBUGFLAGS"} , "else": {"type": "var", "name": "CFLAGS"} } , {"type": "FIELD", "name": "ADD_DEBUGFLAGS"} ] } , "else": {"type": "var", "name": "CFLAGS"} } ] , [ "CXXFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "DEBUGFLAGS"} , "then": {"type": "var", "name": "DEBUGFLAGS"} , "else": {"type": "var", "name": "CXXFLAGS"} } , {"type": "FIELD", "name": "ADD_DEBUGFLAGS"} ] } , "else": {"type": "var", "name": "CXXFLAGS"} } ] , ["deps-fieldnames", ["base", "deps"]] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , ["link-args", {"type": "CALL_EXPRESSION", "name": "link-args-deps"}] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , ["package", {"type": "env", "vars": ["cflags-files", "ldflags-files"]}] , [ "debug-srcs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "let*" , "bindings": [["deps-provider", "debug-srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , "else": {"type": "empty_map"} } ] , [ "debug-hdrs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "compile-deps"} ] } , "else": {"type": "empty_map"} } ] ] , "body": { "type": "RESULT" , "provides": { "type": "env" , "vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "LDFLAGS" , "AR" , "DWP" , "ARFLAGS" , "PATH" , "ENV" , "TOOLCHAIN" , "NON_SYSTEM_TOOLS" , "compile-deps" , "compile-args" , "link-deps" , "link-args" , "package" , "debug-srcs" , "debug-hdrs" ] } } } } , "library": { "doc": ["A C++ library"] , "target_fields": ["srcs", "hdrs", "private-hdrs", "deps", "private-deps", "proto"] , "string_fields": [ "name" , "stage" , "pure C" , "defines" , "private-defines" , "cflags" , "private-cflags" , "private-ldflags" , "pkg-name" ] , "config_vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "AR" , "DWP" , "ENV" , "DEBUG" , "LINT" ] , "implicit": {"defaults": ["defaults"]} , "field_doc": { "name": ["The name of the library (without leading \"lib\" or trailing \".a\")."] , "srcs": ["The source files of the library."] , "hdrs": ["Any public header files of the library."] , "private-hdrs": [ "Any header files that only need to be present when compiling the" , "source files, but are not needed for any consumer of the library." ] , "stage": [ "The logical location of all header and source files, as well as the" , "resulting library file. Individual directory components are joined" , "with \"/\"." ] , "pure C": [ "If non-empty, compile as C sources rather than C++ sources." , "In particular, CC is used to compile rather than CXX (or their" , "respective defaults)." ] , "defines": [ "List of defines set for this target and its consumers." , "Each list entry will be prepended by \"-D\"." ] , "private-defines": [ "List of defines set for source files local to this target." , "Each list entry will be prepended by \"-D\"." ] , "cflags": ["List of compile flags set for this target and its consumers."] , "private-cflags": ["List of compile flags set for source files local to this target."] , "private-ldflags": [ "Additional linker flags for linking external libraries (not built" , "by this tool, typically system libraries)." ] , "deps": ["Any other libraries this library depends upon."] , "private-deps": [ "Any other libraries this library depends upon but does not include" , "in its public headers." ] , "proto": [ "Any [\"proto\", \"library\"] this target depends upon directly." , "The creation of C++ bindings for this proto library as well as of" , "its dependencies will be taken care of (as anonymous targets, so no" , "duplicate work will be carried out, even if the same proto library" , "is used at various places)." ] , "pkg-name": [ "Name to use for pkg-config files. If this field is empty, the field" , "\"name\" is used instead." ] , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "CC": [ "The name of the C compiler to be used (when compiling pure C code)." , "If None, the respective value from [\"CC\", \"defaults\"] will be taken." ] , "CXX": [ "The name of the C++ compiler to be used." , "If None, the respective value from [\"CC\", \"defaults\"] will be taken." ] , "CFLAGS": [ "The flags for CC to be used instead of the default ones." , "For libraries that should be built in a non-standard way; usually" , "adapting the default target [\"CC\", \"defaults\"] is the better" , "choice." ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones." , "For libraries that should be built in a non-standard way; usually" , "adapting the default target [\"CC\", \"defaults\"] is the better" , "choice." ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC." , "For libraries that should be built in a non-standard way; usually" , "adapting the default target [\"CC\", \"defaults\"] is the better" , "choice." ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX." , "For libraries that should be built in a non-standard way; usually" , "adapting the default target [\"CC\", \"defaults\"] is the better" , "choice." ] , "AR": [ "The archive tool to used for creating the library" , "If None, the respective value from [\"CC\", \"defaults\"] will be taken." ] , "DWP": [ "The DWARF format packaging tool to use in debug builds that enable" , "debug fission." , "If None, the respective value from [\"CC\", \"defaults\"] will be taken." ] , "ENV": ["The environment for any action generated."] , "DEBUG": [ "Either a logically false value to disable debugging, or a non-empty" , "map to enable debugging. If debugging is enabled, the following values" , "of the map are used." , "" , "The key \"USE_DEBUG_FISSION\" expects a flag which enables the debug" , "fission mode, but does not add any flags. Explicitly setting it to a" , "false value is needed to enable regular debug mode." , "The key \"FISSION_CONFIG\" expects a map configuring debug fission." , " - subkey \"USE_SPLIT_DWARF\" expects a flag that, if true, adds the" , "-gsplit-dwarf compile flag." , " - subkey \"DWARF_VERSION\" expects a string that adds the" , "-gdwarf- compile flag." , " - subkey \"USE_GDB_INDEX\" expects a flag that, if true, adds the" , "-Wl,--gdb-index linker flag." , " - subkey \"USE_DEBUG_TYPES_SECTION\" expects a flag that, if true," , "adds the -fdebug-types-section compile flag." , "" , "If no compile flags are otherwise configured, [\"-g\"] will be taken." ] , "LINT": [ "Also provide nodes describing compile actions and header files;" , "those can be used by lint rules (doing also the config transition)" , "for additional checks." ] } , "artifacts_doc": ["The actual library (libname.a) staged in the specified directory"] , "runfiles_doc": ["The public headers of this library"] , "provides_doc": { "compile-deps": [ "Map of artifacts specifying any additional files that, besides the runfiles," , "have to be present in compile actions of targets depending on this library" ] , "link-deps": [ "Map of artifacts specifying any additional files that, besides the artifacts," , "have to be present in a link actions of targets depending on this library" ] , "link-args": [ "List of strings that have to be added to the command line for linking actions" , "in targets depending on this library" ] , "debug-srcs": ["Map of all sources needed for debugging."] , "debug-hdrs": ["Map of all additional headers needed for debugging."] } , "anonymous": { "proto-deps": { "target": "proto" , "provider": "proto" , "rule_map": { "library": ["./", "proto", "library"] , "service library": ["./", "proto", "service library"] } } } , "imports": {"artifacts": ["./", "..", "field_artifacts"], "result": "lib result"} , "expression": { "type": "let*" , "bindings": [ ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "DEBUG" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": {"type": "var", "name": "DEBUG"} , "else": {"type": "empty_map"} } ] , ["pure C", {"type": "FIELD", "name": "pure C"}] , [ "cflags" , { "type": "++" , "$1": [ {"type": "FIELD", "name": "cflags"} , { "type": "foreach" , "var": "def" , "range": {"type": "FIELD", "name": "defines"} , "body": {"type": "join", "$1": ["-D", {"type": "var", "name": "def"}]} } ] } ] , [ "private-cflags" , { "type": "++" , "$1": [ {"type": "FIELD", "name": "private-cflags"} , { "type": "foreach" , "var": "def" , "range": {"type": "FIELD", "name": "private-defines"} , "body": {"type": "join", "$1": ["-D", {"type": "var", "name": "def"}]} } ] } ] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "srcs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [["fieldname", "srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [["fieldname", "hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [["fieldname", "private-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , ["private-ldflags", {"type": "FIELD", "name": "private-ldflags"}] , [ "pkg-name" , { "type": "if" , "cond": {"type": "FIELD", "name": "pkg-name"} , "then": {"type": "join", "$1": {"type": "FIELD", "name": "pkg-name"}} , "else": {"type": "var", "name": "name"} } ] , ["public-fieldnames", ["deps", "proto-deps", "defaults"]] , [ "private-fieldnames" , ["deps", "private-deps", "proto-deps", "defaults"] ] ] , "body": {"type": "CALL_EXPRESSION", "name": "result"} } } , "binary": { "doc": ["A binary written in C++"] , "target_fields": ["srcs", "private-hdrs", "private-deps", "private-proto"] , "string_fields": [ "name" , "stage" , "pure C" , "private-defines" , "private-cflags" , "private-ldflags" ] , "config_vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "DWP" , "ENV" , "DEBUG" , "LINT" ] , "implicit": {"defaults": ["defaults"]} , "field_doc": { "name": ["The name of the binary"] , "srcs": ["The source files of the library."] , "private-hdrs": [ "Any header files that need to be present when compiling the" , "source files." ] , "stage": [ "The logical location of all header and source files, as well as the" , "resulting binary file. Individual directory components are joined" , "with \"/\"." ] , "pure C": [ "If non-empty, compile as C sources rather than C++ sources." , "In particular, CC is used to compile rather than CXX" ] , "private-defines": [ "List of defines set for source files local to this target." , "Each list entry will be prepended by \"-D\"." ] , "private-cflags": ["List of compile flags set for source files local to this target."] , "private-ldflags": ["Additional linker flags for linking external libraries."] , "private-deps": ["Any other libraries this binary depends upon."] , "private-proto": [ "Any [\"proto\", \"library\"] this target depends upon directly." , "The creation of C++ bindings for this proto library as well as of" , "is dependencies will be taken care of (as anonymous targets, so no" , "duplicate work will be carried out, even if the same proto library" , "is used at various places)." ] , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "CC": ["The name of the C compiler to be used (when compiling pure C code)"] , "CXX": ["The name of the C++ compiler to be used."] , "CFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX" , "taken from the [\"CC\", \"defaults\"] target" ] , "DWP": [ "The DWARF format packaging tool to use in debug builds that enable" , "debug fission." , "If None, the respective value from [\"CC\", \"defaults\"] will be taken." ] , "ENV": ["The environment for any action generated."] , "DEBUG": [ "Either a logically false value to disable debugging, or a non-empty" , "map to enable debugging. If debugging is enabled, the following values" , "of the map are used." , "" , "The key \"USE_DEBUG_FISSION\" expects a flag which enables the debug" , "fission mode, but does not add any flags. Explicitly setting it to a" , "false value is needed to enable regular debug mode." , "The key \"FISSION_CONFIG\" expects a map configuring debug fission." , " - subkey \"USE_SPLIT_DWARF\" expects a flag that, if true, adds the" , "-gsplit-dwarf compile flag." , " - subkey \"DWARF_VERSION\" expects a string that adds the" , "-gdwarf- compile flag." , " - subkey \"USE_GDB_INDEX\" expects a flag that, if true, adds the" , "-Wl,--gdb-index linker flag." , " - subkey \"USE_DEBUG_TYPES_SECTION\" expects a flag that, if true," , "adds the -fdebug-types-section compile flag." , "" , "If no compile flags are otherwise configured, [\"-g\"] will be taken." ] , "LINT": [ "Also provide nodes describing compile actions and header files;" , "those can be used by lint rules (doing also the config transition)" , "for additional checks." ] } , "artifacts_doc": ["The final binary, staged to the given directory"] , "runfiles_doc": ["None"] , "provides_doc": { "debug-srcs": ["Map of all sources needed for debugging."] , "debug-hdrs": ["Map of all additional headers needed for debugging."] } , "anonymous": { "private-proto-deps": { "target": "private-proto" , "provider": "proto" , "rule_map": { "library": ["./", "proto", "library"] , "service library": ["./", "proto", "service library"] } } } , "imports": {"artifacts": ["./", "..", "field_artifacts"], "bin result": "bin result"} , "expression": { "type": "let*" , "bindings": [ [ "name" , { "type": "assert_non_empty" , "msg": "A non-empty name has to be provided for binaries" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , [ "DEBUG" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": {"type": "var", "name": "DEBUG"} , "else": {"type": "empty_map"} } ] , ["pure C", {"type": "FIELD", "name": "pure C"}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "srcs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [["fieldname", "srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [["fieldname", "private-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-cflags" , { "type": "++" , "$1": [ { "type": "foreach" , "var": "def" , "range": {"type": "FIELD", "name": "private-defines"} , "body": {"type": "join", "$1": ["-D", {"type": "var", "name": "def"}]} } , {"type": "FIELD", "name": "private-cflags"} ] } ] , ["private-ldflags", {"type": "FIELD", "name": "private-ldflags"}] , [ "private-fieldnames" , ["private-deps", "private-proto-deps", "defaults"] ] ] , "body": {"type": "CALL_EXPRESSION", "name": "bin result"} } } , "install-with-deps": { "doc": [ "Install target's artifacts with transitive dependencies. Depending on" , "the target, artifacts and dependencies will be installed to" , "subdirectories \"bin\", \"include\", and \"lib\". For library targets," , "a pkg-config file is generated and provided in \"lib/pkgconfig\"." , "In debug mode, depending on the target, additional artifacts needed for" , "local debugging are gathered and installed, depending on target, to" , "\"work\" and \"include\"." ] , "config_vars": ["PREFIX"] , "target_fields": ["targets"] , "string_fields": ["flat-libs", "prefix", "hdrs-only", "skip-debug-stage"] , "imports": {"install result": "install-with-deps result"} , "field_doc": { "targets": ["Targets to install artifacts from."] , "flat-libs": [ "Install libraries flat to the \"lib\" subdirectory. Be aware that" , "conflicts may occur if any of the (transitive) libraries happen to" , "have the same base name." ] , "prefix": [ "The prefix used for pkg-config files. The path will be made absolute" , "and individual directory components are joined with \"/\". If no" , "prefix is specified, the value from the config variable \"PREFIX\" is" , "taken, with the default value being \"/\"." ] , "hdrs-only": ["Only collect headers from deps (without subdirectory)."] , "skip-debug-stage": [ "If in debug mode, skip installing additional artifacts gathered if no" , "local debugging is needed, but debug targets are nonetheless desired." ] } , "config_doc": { "PREFIX": [ "The absolute path that is used as prefix inside generated pkg-config" , "files. The default value for this variable is \"/\". This variable" , "is ignored if the field \"prefix\" is set." ] } , "artifacts_doc": ["Installed artifacts in subdirectories (\"bin\"/\"include\"/\"lib\")."] , "expression": { "type": "let*" , "bindings": [ ["pc-install-dir", "lib/pkgconfig"] , ["targets", {"type": "FIELD", "name": "targets"}] , [ "prefix" , { "type": "if" , "cond": {"type": "FIELD", "name": "prefix"} , "then": { "type": "join" , "separator": "/" , "$1": {"type": "++", "$1": [[""], {"type": "FIELD", "name": "prefix"}]} } , "else": {"type": "var", "name": "PREFIX", "default": "/"} } ] , ["flat-libs", {"type": "FIELD", "name": "flat-libs"}] , ["hdrs-only", {"type": "FIELD", "name": "hdrs-only"}] , ["skip-debug-stage", {"type": "FIELD", "name": "skip-debug-stage"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "install result"} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/TARGETS000066400000000000000000000003001516554100600222630ustar00rootroot00000000000000{ "defaults": { "type": ["CC", "defaults"] , "CC": ["cc"] , "CXX": ["c++"] , "CFLAGS": [] , "CXXFLAGS": [] , "AR": ["ar"] , "DWP": ["dwp"] , "PATH": ["/bin", "/usr/bin"] } } just-buildsystem-justbuild-b1fb5fa/rules/CC/auto/000077500000000000000000000000001516554100600222065ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/auto/RULES000066400000000000000000001351211516554100600230260ustar00rootroot00000000000000{ "config": { "doc": [ "Generate a C/C++ config header" , "" , "Generate a C/C++ configuration header using defines specified via the" , "target configuration. In the usual case, a target using this rule is" , "configured by depending on it from a target that uses the built-in" , "\"configure\" rule." ] , "field_doc": { "name": ["Name of the header file to generate (incl. file name ext)."] , "guard": ["The include guard. Multiple segments are joined with \"_\"."] , "stage": ["The location of the header. Path segments are joined with \"/\"."] , "hdrs": [ "Additional header files to be available in the include path. Useful" , "for providing additional header files to values given in" , "\"have_{cfile,cxxfile,ctype,cxxtype,csymbol,cxxsymbol}\"." ] , "deps": [ "Additional public header files from targets to be available in the" , "include path. Useful for providing additional header files to values" , "given in \"have_{cfile,cxxfile,ctype,cxxtype,csymbol,cxxsymbol}\"." ] , "defaults": ["The C/C++ toolchain to use"] , "shell defaults": ["The shell toolchain to use"] } , "config_doc": { "CC": [ "The name of the C compiler to be used by checks. If None, the" , "respective value from [\"CC\", \"defaults\"] will be taken." ] , "CXX": [ "The name of the C++ compiler to be used by checks. If None, the" , "respective value from [\"CC\", \"defaults\"] will be taken." ] , "CFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX" , "taken from the [\"CC\", \"defaults\"] target" ] , "ENV": ["The environment for running file/symbol/type/size checks."] , "defines": [ "Set a define to a specific value unless its value is \"null\". Must" , "contain a list of pairs. The first element of each pair is the define" , "name and the second argument is the value to set. Strings must be" , "properly escaped. Defines generated from this field are added last," , "so that they can refer to defines from other \"defines*\", " , "\"have_*\", and \"size_*\" values." ] , "defines1": [ "Set a define to \"1\" unless its value is untrue. Must contain a list" , "of pairs. The first element of each pair is the define name and the" , "second argument is the value." ] , "defines01": [ "Set a define to \"0\" or \"1\" depending on its value being true." , "Must contain a list of pairs. The first element of each pair is the" , "define name and the second argument is the value." ] , "have_cfile": [ "Set a define to \"1\" if the specified C header is in the include" , "path. Must contain a list of pairs. The first element of each pair is" , "the define name and the second argument is the C header file name." ] , "have_cxxfile": [ "Set a define to \"1\" if the specified C++ header is in the include" , "path. Must contain a list of pairs. The first element of each pair is" , "the define name and the second argument is the C++ header file name." ] , "have_ctype": [ "Set a define to \"1\" if the specified C type is defined. Must" , "contain a list of pairs. The first element of each pair is the define" , "name and the second argument is name of the C type. If the specified" , "C type is not a built-in type, additionally the headers" , "\"sys/types.h\", \"stdint.h\", and \"stddef.h\" are checked as well." ] , "have_cxxtype": [ "Set a define to \"1\" if the specified C++ type is defined. Must" , "contain a list of pairs. The first element of each pair is the define" , "name and the second argument is name of the C++ type. If the specified" , "C++ type is not a built-in type, additionally the headers" , "\"sys/types.h\", \"stdint.h\", and \"stddef.h\" are checked as well." ] , "have_csymbol": [ "Set a define to \"1\" if the specified C symbol is defined by one of" , "the specified headers in the include path. Must contain a list of" , "pairs. The first element of each pair is the define name and the" , "second argument is another pair. This pair's first value is the C" , "symbol to search for and the second value is a list with the header" , "file names to consider for searching. If the header file defines the" , "symbol as a macro it is considered available and assumed to work." ] , "have_cxxsymbol": [ "Set a define to \"1\" if the specified C++ symbol is defined by one of" , "the specified headers in the include path. Must contain a list of" , "pairs. The first element of each pair is the define name and the" , "second argument is another pair. This pair's first value is the C++" , "symbol to search for and the second value is a list with the header" , "file names to consider for searching. If the header file defines the" , "symbol as a macro it is considered available and assumed to work." ] , "size_ctype": [ "Set a define to size of the specified C type. Must contain a list of" , "pairs. The first element of each pair is the define name and the" , "second argument is another pair. This pair's first value is the C" , "type to check for and the second value is a list with possible sizes" , "as numbers. If none of the specified sizes matches, the action fails." ] , "size_cxxtype": [ "Set a define to size of the specified C++ type. Must contain a list of" , "pairs. The first element of each pair is the define name and the" , "second argument is another pair. This pair's first value is the C++" , "type to check for and the second value is a list with possible sizes" , "as numbers. If none of the specified sizes matches, the action fails." ] } , "string_fields": ["name", "stage", "guard"] , "target_fields": ["hdrs", "deps"] , "config_vars": [ "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "defines" , "defines1" , "defines01" , "have_cfile" , "have_cxxfile" , "have_ctype" , "have_cxxtype" , "have_csymbol" , "have_cxxsymbol" , "size_ctype" , "size_cxxtype" ] , "imports": { "artifacts": ["./", "../..", "field_artifacts"] , "compile-deps": ["./", "..", "compile-deps"] , "compiler-cc": ["./", "..", "compiler-cc"] , "compiler-cxx": ["./", "..", "compiler-cxx"] , "flags-cc": ["./", "..", "flags-cc"] , "flags-cxx": ["./", "..", "flags-cxx"] , "default-ENV": ["./", "..", "default-ENV"] , "default-PATH": ["./", "..", "default-PATH"] , "default-TOOLCHAIN": ["./", "..", "default-TOOLCHAIN"] , "default-NON_SYSTEM_TOOLS": ["./", "..", "default-NON_SYSTEM_TOOLS"] , "map_provider": ["./", "../..", "field_map_provider"] , "list_provider": ["./", "../..", "field_list_provider"] , "sh": ["./", "../../shell", "sh"] , "sh-PATH": ["./", "../../shell", "PATH"] } , "implicit": { "defaults": [["./", "..", "defaults"]] , "shell defaults": [["./", "../../shell", "defaults"]] } , "expression": { "type": "let*" , "bindings": [ ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "guard" , { "type": "assert_non_empty" , "msg": "Config header include guard may not be empty" , "$1": { "type": "join" , "separator": "_" , "$1": {"type": "FIELD", "name": "guard"} } } ] , [ "includes" , { "type": "to_subdir" , "subdir": "include" , "$1": { "type": "disjoint_map_union" , "msg": "Includes may not overlap" , "$1": [ { "type": "let*" , "bindings": [["fieldname", "hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } , { "type": "let*" , "bindings": [["deps-fieldnames", ["deps"]]] , "body": {"type": "CALL_EXPRESSION", "name": "compile-deps"} } ] } } ] , ["TOOLCHAIN_DIR", "toolchain"] , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] , [ "shell TOOLCHAIN" , { "type": "let*" , "bindings": [ ["fieldname", "shell defaults"] , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } ] , [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "Shell and CC toolchain must not conflict" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "var", "name": "shell TOOLCHAIN"} ] } ] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "sh" , { "type": "let*" , "bindings": [["fieldname", "shell defaults"]] , "body": {"type": "CALL_EXPRESSION", "name": "sh"} } ] , [ "bin dirs" , { "type": "let*" , "bindings": [["fieldname", "shell defaults"], ["provider", "bin dirs"]] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } ] , [ "bin dirs" , { "type": "foreach" , "range": {"type": "var", "name": "bin dirs"} , "body": { "type": "join" , "$1": [ "./" , {"type": "var", "name": "TOOLCHAIN_DIR"} , "/" , {"type": "var", "name": "_"} ] } } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , ["CC", {"type": "CALL_EXPRESSION", "name": "compiler-cc"}] , ["CXX", {"type": "CALL_EXPRESSION", "name": "compiler-cxx"}] , ["CFLAGS", {"type": "CALL_EXPRESSION", "name": "flags-cc"}] , ["CXXFLAGS", {"type": "CALL_EXPRESSION", "name": "flags-cxx"}] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } , { "type": "let*" , "bindings": [["fieldname", "shell defaults"]] , "body": {"type": "CALL_EXPRESSION", "name": "sh-PATH"} } , {"type": "var", "name": "bin dirs"} ] } } } ] } ] , [ "c.flags" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": {"type": "var", "name": "CFLAGS"} } } ] , [ "cxx.flags" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": {"type": "var", "name": "CXXFLAGS"} } } ] , [ "file_check.sh" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -eu" , "[ $# -ge 4 ]" , "CC=$1" , "LANG=$2" , "DEF=$3" , "HDR=$4" , "DEFINE=\"/* #undef $DEF */\"" , "echo \"#include \\\"$HDR\\\"\" > test.$LANG" , "if $CC @$LANG.flags -c test.$LANG -I ./include 2>/dev/null; then DEFINE=\"#define $DEF 1\"; fi" , "echo \"$DEFINE\n\" > out.def" ] } } ] , [ "type_check.sh" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -eu" , "[ $# -ge 4 ]" , "CC=$1" , "LANG=$2" , "DEF=$3" , "TYPE=$4" , "INC=\"\"" , "DEFINE=\"/* #undef $DEF */\"" , "for HDR in \"\" \"sys/types.h\" \"stdint.h\" \"stddef.h\"; do" , " if [ -n \"$HDR\" ]; then INC=\"#include \\\"$HDR\\\"\"; fi" , " cat > test.$LANG << EOF" , "$INC" , "$TYPE* __test;" , "EOF" , " if $CC @$LANG.flags -c test.$LANG -I ./include 2>/dev/null; then" , " DEFINE=\"#define $DEF 1\"" , " break" , " fi" , "done" , "echo \"$DEFINE\n\" > out.def" ] } } ] , [ "symbol_check.sh" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -eu" , "[ $# -ge 4 ]" , "CC=$1" , "shift" , "LANG=$1" , "shift" , "DEF=$1" , "shift" , "SYMBOL=$1" , "shift" , "DEFINE=\"/* #undef $DEF */\"" , "for INC in \"$@\"; do" , " cat > test_symbol.$LANG << EOF" , "#include \"$INC\"" , "void* __test = &$SYMBOL;" , "EOF" , " if $CC @$LANG.flags -c test_symbol.$LANG -I ./include 2>/dev/null; then" , " DEFINE=\"#define $DEF 1\"" , " break" , " fi" , " cat > test_macro.$LANG << EOF" , "#include \"$INC\"" , "#ifndef $SYMBOL" , "#error not defined as macro" , "#endif" , "EOF" , " if $CC @$LANG.flags -c test_macro.$LANG -I ./include 2>/dev/null; then" , " DEFINE=\"#define $DEF 1\"" , " break" , " fi" , "done" , "echo \"$DEFINE\n\" > out.def" ] } } ] , [ "size_check.sh" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -eu" , "[ $# -ge 4 ]" , "CC=$1" , "shift" , "LANG=$1" , "shift" , "DEF=$1" , "shift" , "TYPE=$1" , "shift" , "INC=\"\"" , "for HDR in \"\" \"sys/types.h\" \"stdint.h\" \"stddef.h\"; do" , " if [ -n \"$HDR\" ]; then INC=\"#include \\\"$HDR\\\"\"; fi" , " for SIZE in \"$@\"; do" , " SIZE=$(printf %.0f $SIZE)" , " cat > test.$LANG << EOF" , "$INC" , "char __test[(sizeof($TYPE) == $SIZE) ? 1 : -1];" , "EOF" , " if $CC @$LANG.flags -c test.$LANG -I ./include 2>/dev/null; then" , " DEFINE=\"#define $DEF $SIZE\"" , " echo \"$DEFINE\n\" > out.def" , " exit 0" , " fi" , " done" , "done" , "exit 1" ] } } ] , [ "guard.def" , { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ { "type": "join" , "separator": " " , "$1": ["#ifndef", {"type": "var", "name": "guard"}] } , { "type": "join" , "separator": " " , "$1": ["#define", {"type": "var", "name": "guard"}] } , "\n\n" ] } } ] , [ "plain.def" , { "type": "BLOB" , "data": { "type": "join" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "defines", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'defines' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "join" , "separator": " " , "$1": { "type": "case*" , "expr": {"type": "var", "name": "val"} , "case": [ [ null , [ "/* #undef" , {"type": "var", "name": "def"} , "*/\n\n" ] ] ] , "default": [ "#define" , {"type": "var", "name": "def"} , { "type": "join" , "$1": [{"type": "var", "name": "val"}, "\n\n"] } ] } } } } } } ] , [ "int1.def" , { "type": "BLOB" , "data": { "type": "join" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "defines1", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'defines1' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "join" , "separator": " " , "$1": { "type": "if" , "cond": {"type": "var", "name": "val"} , "then": ["#define", {"type": "var", "name": "def"}, "1\n\n"] , "else": ["/* #undef", {"type": "var", "name": "def"}, "*/\n\n"] } } } } } } ] , [ "int01.def" , { "type": "BLOB" , "data": { "type": "join" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "defines01", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'defines01' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "join" , "separator": " " , "$1": [ "#define" , {"type": "var", "name": "def"} , { "type": "if" , "cond": {"type": "var", "name": "val"} , "then": "1\n\n" , "else": "0\n\n" } ] } } } } } ] , [ "cfile-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_cfile", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_cfile' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["file_check.sh", "c.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "./file_check.sh" , {"type": "var", "name": "CC"} , "c" , {"type": "var", "name": "def"} , {"type": "var", "name": "val"} ] , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "cxxfile-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_cxxfile", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_cxxfile' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["file_check.sh", "cxx.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "./file_check.sh" , {"type": "var", "name": "CXX"} , "cxx" , {"type": "var", "name": "def"} , {"type": "var", "name": "val"} ] , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "ctype-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_ctype", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_ctype' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "type" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["type_check.sh", "c.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "./type_check.sh" , {"type": "var", "name": "CC"} , "c" , {"type": "var", "name": "def"} , {"type": "var", "name": "type"} ] , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "cxxtype-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_cxxtype", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_cxxtype' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "type" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["type_check.sh", "cxx.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "./type_check.sh" , {"type": "var", "name": "CXX"} , "cxx" , {"type": "var", "name": "def"} , {"type": "var", "name": "type"} ] , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "csymbol-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_csymbol", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_csymbol' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "sym, hdrs" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] , [ "sym" , { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "sym, hdrs"} } ] , [ "hdrs" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "sym, hdrs"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["symbol_check.sh", "c.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": { "type": "++" , "$1": [ [ {"type": "var", "name": "sh"} , "./symbol_check.sh" , {"type": "var", "name": "CC"} , "c" , {"type": "var", "name": "def"} , {"type": "var", "name": "sym"} ] , {"type": "var", "name": "hdrs"} ] } , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "cxxsymbol-defs" , { "type": "foreach" , "range": {"type": "var", "name": "have_cxxsymbol", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_csymbol' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "sym, hdrs" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] , [ "sym" , { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "sym, hdrs"} } ] , [ "hdrs" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "sym, hdrs"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["symbol_check.sh", "cxx.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": { "type": "++" , "$1": [ [ {"type": "var", "name": "sh"} , "./symbol_check.sh" , {"type": "var", "name": "CXX"} , "cxx" , {"type": "var", "name": "def"} , {"type": "var", "name": "sym"} ] , {"type": "var", "name": "hdrs"} ] } , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "csize-defs" , { "type": "foreach" , "range": {"type": "var", "name": "size_ctype", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_csymbol' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "type, sizes" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] , [ "type" , { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "type, sizes"} } ] , [ "sizes" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "type, sizes"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["size_check.sh", "c.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": { "type": "++" , "$1": [ [ {"type": "var", "name": "sh"} , "./size_check.sh" , {"type": "var", "name": "CC"} , "c" , {"type": "var", "name": "def"} , {"type": "var", "name": "type"} ] , { "type": "foreach" , "var": "size" , "range": {"type": "var", "name": "sizes"} , "body": { "type": "json_encode" , "$1": {"type": "var", "name": "size"} } } ] } , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , [ "cxxsize-defs" , { "type": "foreach" , "range": {"type": "var", "name": "size_cxxtype", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "def" , { "type": "assert_non_empty" , "msg": "Define name in 'have_csymbol' may not be empty" , "$1": { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } } ] , [ "type, sizes" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] , [ "type" , { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "type, sizes"} } ] , [ "sizes" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "type, sizes"} } ] ] , "body": { "type": "lookup" , "key": "out.def" , "map": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "env", "vars": ["size_check.sh", "cxx.flags"]} , {"type": "var", "name": "includes"} ] } , "cmd": { "type": "++" , "$1": [ [ {"type": "var", "name": "sh"} , "./size_check.sh" , {"type": "var", "name": "CXX"} , "cxx" , {"type": "var", "name": "def"} , {"type": "var", "name": "type"} ] , { "type": "foreach" , "var": "size" , "range": {"type": "var", "name": "sizes"} , "body": { "type": "json_encode" , "$1": {"type": "var", "name": "size"} } } ] } , "env": {"type": "var", "name": "ENV"} , "outs": ["out.def"] } } } } ] , ["end.def", {"type": "BLOB", "data": "\n#endif\n"}] , [ "definitions" , { "type": "enumerate" , "$1": { "type": "++" , "$1": [ [ {"type": "var", "name": "guard.def"} , {"type": "var", "name": "int1.def"} , {"type": "var", "name": "int01.def"} ] , {"type": "var", "name": "cfile-defs"} , {"type": "var", "name": "cxxfile-defs"} , {"type": "var", "name": "ctype-defs"} , {"type": "var", "name": "cxxtype-defs"} , {"type": "var", "name": "csymbol-defs"} , {"type": "var", "name": "cxxsymbol-defs"} , {"type": "var", "name": "csize-defs"} , {"type": "var", "name": "cxxsize-defs"} , [ {"type": "var", "name": "plain.def"} , {"type": "var", "name": "end.def"} ] ] } } ] , [ "outfile" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "TOOLCHAIN"} , {"type": "var", "name": "definitions"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "-c" , { "type": "join" , "separator": " " , "$1": [ "cat" , { "type": "join_cmd" , "$1": { "type": "keys" , "$1": {"type": "var", "name": "definitions"} } } , "> out" ] } ] , "outs": ["out"] , "env": {"type": "var", "name": "ENV"} } ] , [ "outfile" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": { "type": "lookup" , "key": "out" , "map": {"type": "var", "name": "outfile"} } } } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "outfile"} , "runfiles": {"type": "var", "name": "outfile"} } } } , "config_file": { "doc": [ "Generate a C/C++ config header from a given template" , "" , "Generate a C/C++ configuration header using defines specified via the" , "target configuration. In the usual case, a target using this rule is" , "configured by depending on it from a target that uses the built-in" , "\"configure\" rule." , "" , "The actual generation of the header file from the template" , "is done by the implicit dependency on the \"runner\" target which" , "can be changed globally by setting this target in the" , "target layer of this repository." ] , "field_doc": { "output": [ "Name of the header file to generate (incl. file name ext). Components are joined with /." ] , "input": ["The input configuration file, used as template."] , "magic_string": [ "The magic string (e.g., \"cmakedefine\") which identifies in which line" , "we have to \"#define\" or \"#undef\" variables according to what is" , "defined in the config field \"defines\"." ] , "@only": ["If set, only replace @VAR@ and not ${VAR}"] , "runner": ["The program generating the header file from the template."] } , "config_doc": { "defines": [ "Set a define to a specific value unless its value is \"null\". Must" , "contain a list of pairs. The first element of each pair is the define" , "name and the second argument is the value to set. Strings must be" , "properly escaped. Defines generated from this field are added last," , "so that they can refer to defines from other \"defines*\" values." ] } , "string_fields": ["magic_string", "@only", "output"] , "target_fields": ["input"] , "config_vars": ["defines"] , "imports": { "stage_singleton_field": ["", "stage_singleton_field"] , "default-PATH": ["./", "..", "default-PATH"] } , "implicit": {"runner": ["runner"], "defaults": [["./", "..", "defaults"]]} , "expression": { "type": "let*" , "bindings": [ [ "runner" , { "type": "let*" , "bindings": [["fieldname", "runner"], ["location", "runner"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage_singleton_field"} } ] , [ "dict-defines" , { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "defines", "default": []} , "var": "pair" , "body": { "type": "let*" , "bindings": [ [ "key" , { "type": "[]" , "index": 0 , "list": {"type": "var", "name": "pair"} } ] , [ "val" , { "type": "[]" , "index": -1 , "list": {"type": "var", "name": "pair"} } ] ] , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "key"} , "value": {"type": "var", "name": "val"} } } } } ] , [ "magic_string" , { "type": "assert_non_empty" , "msg": "A non-empty string has to be provided for magic_string" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "magic_string"}} } ] , [ "@only" , { "type": "if" , "cond": {"type": "FIELD", "name": "@only"} , "then": "true" , "else": "false" } ] , [ "param-blob" , { "type": "singleton_map" , "key": "param-file" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": {"type": "var", "name": "dict-defines"} } } } ] , [ "input-blob" , { "type": "let*" , "bindings": [["fieldname", "input"], ["location", "input-file"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage_singleton_field"} } ] , ["PATH", {"type": "CALL_EXPRESSION", "name": "default-PATH"}] , [ "ENV" , { "type": "if" , "cond": {"type": "var", "name": "PATH"} , "then": { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": {"type": "var", "name": "PATH"} } } , "else": {"type": "empty_map"} } ] , [ "outfile" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "param-blob"} , {"type": "var", "name": "input-blob"} , {"type": "var", "name": "runner"} ] } , "cmd": [ "./runner" , "input-file" , "param-file" , {"type": "var", "name": "magic_string"} , {"type": "var", "name": "@only"} ] , "env": {"type": "var", "name": "ENV"} , "outs": ["out"] } ] , [ "outfile" , { "type": "singleton_map" , "key": { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "output"} } , "value": { "type": "lookup" , "key": "out" , "map": {"type": "var", "name": "outfile"} } } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "outfile"} , "runfiles": {"type": "var", "name": "outfile"} } } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/auto/TARGETS000066400000000000000000000000031516554100600232330ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/rules/CC/auto/runner000077500000000000000000000123241516554100600234470ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import re from sys import argv from typing import Union def get_tokens(line: str, magic_string: str) -> tuple[Union[re.Match[str], None], bool]: """Tokenize lines (strings) like the following #cmakedefine FOO bar #cmakedefine FOO @FOO@ #cmakedefine FOO ${FOO} #cmakedefine01 FOO where "cmakedefine" is the magic_string. Let us name "FOO" as the token_key, and the corresponding value (i.e., bar, @FOO@, ${FOO}, the empty string) as the token_value. The function handles any combination of spaces around the magic_string, the token_key and the token_value. """ x = re.search( r"#(.*)(" + magic_string + r"[01]*" + r")([\s]*)([a-zA-Z0-9_]+)([\s]*)(.*)", line, ) if x: return x, x.groups()[1] == f"{magic_string}01" return None, False def handle01(line: str, tokens: re.Match[str], defined: bool) -> str: groups = tokens.groups() return re.sub( tokens.group()[1:], groups[0] # spaces + "define" + groups[2] # spaces + groups[3] # token_key + " " + str(1 if defined else 0), line, ) def undefine(tokens: re.Match[str]) -> str: groups = tokens.groups() return "/* #" + groups[0] + "undef" + groups[2] + groups[3] + " */" def replace_value(tokens: re.Match[str], key: str, value: str) -> str: groups = tokens.groups() return f"#{groups[0]}define{groups[2]}{key}{groups[4]}{value}" def compute_value(token_value: str, at_only: bool, param: dict[str, str]): # example of possible token_values # - foo (a simple string) # - @FOO@ # - ${FOO} # - any combination of the above def replace_pattern_in_string( pattern: str, line: str, param: dict[str, str] ) -> str: def get_value_for_match(match: re.Match[str]) -> str: key = match.group(1) return param.get(key, "") return re.sub(pattern, get_value_for_match, line) token_value = replace_pattern_in_string(r"@([A-Za-z0-9_]+)@", token_value, param) if at_only: return token_value return replace_pattern_in_string(r"\${([A-Za-z0-9_]+)}", token_value, param) if __name__ == "__main__": input_file = argv[1] param_file = argv[2] magic_string = argv[3] at_only = argv[4] == "true" with open(param_file) as f: param = json.loads(f.read()) # In many cases, CMake simply defines some variables (without any associated # value). We handle this situation by assigning to the boolean True the empty # string. Note that no False value should be found, because the right way to set # a variable to False in the TARGETS file is to *do not mention* that variable # at all. # If a value is deliberately set to null, we will drop that key drop_keys: list[str] = [] for k, v in param.items(): if isinstance(v, bool): param[k] = "" if v == None: drop_keys.append(k) for k in drop_keys: del param[k] with open(input_file) as i: with open("out", "w") as o: for line in i.readlines(): # drop trailing '\n' line = line[:-1] tokens, is_01 = get_tokens(line, magic_string) # no magic string if not tokens: # it can be a simple comment, or a line without the magic string but with the @KEY@ or ${KEY} pattern line = compute_value(line, at_only, param) print(line, file=o) continue # line contains magic_string groups = tokens.groups() token_key: str = groups[3] if is_01: line = handle01(line, tokens, token_key in param) print(line, file=o) continue if token_key not in param: line = undefine(tokens) print(line, file=o) continue # we are in one of this situations # cmakedefine FOO # cmakedefine FOO "foo" # cmakedefine FOO @FOO@${FOO}foo # i.e., the token_value can be any combination of keys (defined # as @key@ or ${key}) and strings # therefore, we need to further tokenize the token_value # it is convenient to first tokenize token_value: str = groups[5] value = compute_value(token_value, at_only, param) line = replace_value(tokens, token_key, value) print(line, file=o) just-buildsystem-justbuild-b1fb5fa/rules/CC/pkgconfig/000077500000000000000000000000001516554100600232055ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/pkgconfig/EXPRESSIONS000066400000000000000000000161361516554100600247610ustar00rootroot00000000000000{ "pkgconfig result": { "vars": ["ENV", "name", "args", "stage"] , "imports": { "default-ENV": ["./", "..", "default-ENV"] , "default-PATH": ["./", "..", "default-PATH"] , "stage": ["", "stage_singleton_field"] , "map_provider": ["", "field_map_provider"] , "sh": ["shell", "sh"] , "sh-PATH": ["shell", "PATH"] } , "expression": { "type": "let*" , "bindings": [ [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "sh-PATH" , { "type": "let*" , "bindings": [["fieldname", "shell defaults"]] , "body": {"type": "CALL_EXPRESSION", "name": "sh-PATH"} } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , {"type": "var", "name": "sh-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , ["TOOLCHAIN_DIR", "toolchain"] , [ "shell TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "shell defaults"] , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } ] , [ "sh" , { "type": "let*" , "bindings": [["fieldname", "shell defaults"]] , "body": {"type": "CALL_EXPRESSION", "name": "sh"} } ] , [ "cflags-filename" , {"type": "join", "$1": [{"type": "var", "name": "name"}, ".cflags"]} ] , [ "cflags-files" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "ACTION" , "inputs": {"type": "var", "name": "shell TOOLCHAIN"} , "cmd": [ {"type": "var", "name": "sh"} , "-c" , { "type": "join" , "separator": " " , "$1": { "type": "++" , "$1": [ ["pkg-config"] , [ { "type": "join_cmd" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "args", "default": []} , ["--cflags", {"type": "var", "name": "name"}] ] } } ] , [">"] , [ { "type": "join_cmd" , "$1": [{"type": "var", "name": "cflags-filename"}] } ] ] } } ] , "env": {"type": "var", "name": "ENV"} , "outs": [{"type": "var", "name": "cflags-filename"}] } } ] , [ "compile-args" , { "type": "foreach_map" , "var_key": "flag-file" , "range": {"type": "var", "name": "cflags-files"} , "body": {"type": "join", "$1": ["@", {"type": "var", "name": "flag-file"}]} } ] , [ "ldflags-filename" , {"type": "join", "$1": [{"type": "var", "name": "name"}, ".ldflags"]} ] , [ "ldflags-files" , { "type": "ACTION" , "inputs": {"type": "var", "name": "shell TOOLCHAIN"} , "cmd": [ {"type": "var", "name": "sh"} , "-c" , { "type": "join" , "separator": " " , "$1": { "type": "++" , "$1": [ ["pkg-config"] , [ { "type": "join_cmd" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "args", "default": []} , ["--libs", {"type": "var", "name": "name"}] ] } } ] , [">", "ldflags.raw"] ] } } ] , "env": {"type": "var", "name": "ENV"} , "outs": ["ldflags.raw"] } ] , [ "add_rpath" , { "type": "let*" , "bindings": [["fieldname", "add_rpath"], ["location", "add_rpath"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "ldflags-files" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "add_rpath"} , {"type": "var", "name": "ldflags-files"} , {"type": "var", "name": "shell TOOLCHAIN"} ] } , "cmd": [ {"type": "var", "name": "sh"} , "-c" , { "type": "join" , "separator": " " , "$1": [ "./add_rpath $(cat ldflags.raw)" , ">" , { "type": "join_cmd" , "$1": [{"type": "var", "name": "ldflags-filename"}] } ] } ] , "env": {"type": "var", "name": "ENV"} , "outs": [{"type": "var", "name": "ldflags-filename"}] } } ] , [ "link-args" , { "type": "foreach_map" , "var_key": "flag-file" , "range": {"type": "var", "name": "ldflags-files"} , "body": {"type": "join", "$1": ["@", {"type": "var", "name": "flag-file"}]} } ] , ["package", {"type": "env", "vars": ["cflags-files", "ldflags-files"]}] , ["compile-deps", {"type": "empty_map"}] , ["link-deps", {"type": "empty_map"}] ] , "body": { "type": "RESULT" , "provides": { "type": "env" , "vars": ["compile-deps", "compile-args", "link-deps", "link-args", "package"] } } } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/pkgconfig/RULES000066400000000000000000000033551516554100600240300ustar00rootroot00000000000000{ "system_library": { "doc": ["A system library via pkg-config"] , "string_fields": ["name", "args", "stage"] , "implicit": { "defaults": [["./", "..", "defaults"]] , "shell defaults": [["shell", "defaults"]] , "add_rpath": ["add_rpath"] } , "config_vars": ["PKG_CONFIG_ARGS", "ENV"] , "field_doc": { "name": ["The pkg-config name of the library."] , "args": [ "Additional pkg-config arguments (e.g., \"--define-prefix\" or" , "\"--static\"), appended to the config variable \"PKG_CONFIG_ARGS\"." ] , "stage": ["The stage of the internally created flag files."] , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "PKG_CONFIG_ARGS": [ "Additional pkg-config arguments (e.g., \"--define-prefix\" or" , "\"--static\")." ] , "ENV": [ "The environment for any action generated. May contain colon-separated" , "\"PKG_CONFIG_PATH\" for looking up pkg-config files." ] } , "imports": {"pkgconfig result": "pkgconfig result"} , "expression": { "type": "let*" , "bindings": [ [ "name" , { "type": "assert_non_empty" , "msg": "system_library requires non-empty name" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , [ "args" , { "type": "++" , "$1": [ {"type": "var", "name": "PKG_CONFIG_ARGS", "default": []} , {"type": "FIELD", "name": "args"} ] } ] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "pkgconfig result"} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/pkgconfig/TARGETS000066400000000000000000000000031516554100600242320ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/rules/CC/pkgconfig/add_rpath000077500000000000000000000014161516554100600250630ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. for x in "$@" do echo -n '' "$x" case $x in -L*) echo -n '' -Xlinker -rpath -Xlinker "$(expr "$x" : "..\(.*\)")" ;; esac done echo just-buildsystem-justbuild-b1fb5fa/rules/CC/proto/000077500000000000000000000000001516554100600224015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/proto/EXPRESSIONS000066400000000000000000000370251516554100600241550ustar00rootroot00000000000000{ "default-PROTOC": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "PROTOC"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-PROTOCFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "PROTOCFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-LDFLAGS": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "LDFLAGS"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-GRPC_PLUGIN": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "GRPC_PLUGIN"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-ENV": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "../..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "ENV"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-PATH": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "PATH"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-TOOLCHAIN": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "../..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "TOOLCHAIN"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-NON_SYSTEM_TOOLS": { "vars": ["defaults-transition"] , "expression": { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "proto-defaults"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "provider": "NON_SYSTEM_TOOLS" , "transition": { "type": "var" , "name": "defaults-transition" , "default": {"type": "empty_map"} } , "default": {"type": "empty_map"} } } } } , "protoc-deps": { "imports": {"map_provider": ["./", "../..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [["fieldname", "deps"], ["provider", "protoc-deps"]] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "protoc-compile": { "vars": [ "CXX" , "CXXFLAGS" , "ADD_CXXFLAGS" , "AR" , "ENV" , "DEBUG" , "name" , "stage" , "service support" , "public-fieldnames" , "private-fieldnames" ] , "imports": { "stage": ["", "stage_singleton_field"] , "result": ["./", "..", "lib result"] , "runfiles": ["./", "../..", "field_runfiles"] , "artifacts_list": ["./", "../..", "field_artifacts_list"] , "protoc-deps": "protoc-deps" , "default-PROTOC": "default-PROTOC" , "default-PROTOCFLAGS": "default-PROTOCFLAGS" , "default-LDFLAGS": "default-LDFLAGS" , "default-GRPC_PLUGIN": "default-GRPC_PLUGIN" , "default-ENV": "default-ENV" , "default-PATH": "default-PATH" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" } , "expression": { "type": "let*" , "bindings": [ ["pure C", false] , ["TOOLCHAIN_DIR", "toolchain"] , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , [ "PROTOC" , { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "PROTOC" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-PROTOC"} ] } } ] , [ "GRPC_PLUGIN" , { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "GRPC_PLUGIN" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-GRPC_PLUGIN"} ] } } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "protoc-ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , ["protoc-deps", {"type": "CALL_EXPRESSION", "name": "protoc-deps"}] , [ "proto srcs" , { "type": "disjoint_map_union" , "msg": "Sources may not conflict" , "$1": { "type": "let*" , "bindings": [["fieldname", "srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } ] , [ "all proto srcs" , { "type": "disjoint_map_union" , "msg": "Conflict with proto files of dependencies" , "$1": [ {"type": "var", "name": "protoc-deps"} , {"type": "var", "name": "proto srcs"} ] } ] , [ "staged srcs" , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "proto srcs"} } ] , [ "staged all proto srcs" , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "all proto srcs"} } ] , [ "outs" , { "type": "++" , "$1": { "type": "foreach" , "var": "f" , "range": {"type": "keys", "$1": {"type": "var", "name": "staged srcs"}} , "body": { "type": "++" , "$1": [ [ { "type": "change_ending" , "$1": {"type": "var", "name": "f"} , "ending": ".pb.h" } , { "type": "change_ending" , "$1": {"type": "var", "name": "f"} , "ending": ".pb.cc" } ] , { "type": "if" , "cond": {"type": "var", "name": "service support"} , "then": [ { "type": "change_ending" , "$1": {"type": "var", "name": "f"} , "ending": ".grpc.pb.h" } , { "type": "change_ending" , "$1": {"type": "var", "name": "f"} , "ending": ".grpc.pb.cc" } ] , "else": [] } ] } } } ] , [ "cmd" , { "type": "++" , "$1": [ [{"type": "var", "name": "PROTOC"}] , {"type": "CALL_EXPRESSION", "name": "default-PROTOCFLAGS"} , ["--proto_path=work", "--cpp_out=work"] , { "type": "if" , "cond": {"type": "var", "name": "service support"} , "then": [ "--grpc_out=work" , { "type": "join" , "$1": [ "--plugin=protoc-gen-grpc=" , {"type": "var", "name": "GRPC_PLUGIN"} ] } ] , "else": [] } , {"type": "keys", "$1": {"type": "var", "name": "staged srcs"}} ] } ] , [ "generated" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "staged all proto srcs"} , {"type": "var", "name": "TOOLCHAIN"} ] } , "outs": {"type": "var", "name": "outs"} , "cmd": {"type": "var", "name": "cmd"} , "env": {"type": "var", "name": "protoc-ENV"} } ] , [ "srcs" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "name" , "range": {"type": "keys", "$1": {"type": "var", "name": "proto srcs"}} , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".pb.cc" } , "value": { "type": "lookup" , "map": {"type": "var", "name": "generated"} , "key": { "type": "join" , "$1": [ "work/" , { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".pb.cc" } ] } } } , { "type": "if" , "cond": {"type": "var", "name": "service support"} , "then": { "type": "singleton_map" , "key": { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".grpc.pb.cc" } , "value": { "type": "lookup" , "map": {"type": "var", "name": "generated"} , "key": { "type": "join" , "$1": [ "work/" , { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".grpc.pb.cc" } ] } } } , "else": {"type": "empty_map"} } ] } } } ] , [ "hdrs" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "name" , "range": {"type": "keys", "$1": {"type": "var", "name": "proto srcs"}} , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".pb.h" } , "value": { "type": "lookup" , "map": {"type": "var", "name": "generated"} , "key": { "type": "join" , "$1": [ "work/" , { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".pb.h" } ] } } } , { "type": "if" , "cond": {"type": "var", "name": "service support"} , "then": { "type": "singleton_map" , "key": { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".grpc.pb.h" } , "value": { "type": "lookup" , "map": {"type": "var", "name": "generated"} , "key": { "type": "join" , "$1": [ "work/" , { "type": "change_ending" , "$1": {"type": "var", "name": "name"} , "ending": ".grpc.pb.h" } ] } } } , "else": {"type": "empty_map"} } ] } } } ] , ["private-hdrs", {"type": "empty_map"}] , [ "extra-provides" , { "type": "singleton_map" , "key": "protoc-deps" , "value": {"type": "var", "name": "all proto srcs"} } ] , ["cflags", []] , ["private-cflags", []] , [ "private-ldflags" , {"type": "CALL_EXPRESSION", "name": "default-LDFLAGS"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "result"} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/proto/RULES000066400000000000000000000317071516554100600232260ustar00rootroot00000000000000{ "defaults": { "doc": [ "A rule to provide protoc/GRPC defaults." , "Used to implement [\"CC/proto\", \"defaults\"] for CC proto libraries" , "and [\"CC/proto\", \"service defaults\"] for CC proto service libraries" , "(GRPC)." ] , "target_fields": ["base", "toolchain", "deps"] , "string_fields": [ "PROTOC" , "PROTOCFLAGS" , "ADD_PROTOCFLAGS" , "LDFLAGS" , "ADD_LDFLAGS" , "GRPC_PLUGIN" , "PATH" ] , "config_vars": ["ARCH", "HOST_ARCH", "DEBUG"] , "field_doc": { "base": [ "Other targets (using the same rule) to inherit values from. If" , "multiple targets are specified, for values that are overwritten (see" , "documentation of other fields) the last specified value wins." ] , "toolchain": [ "Optional toolchain directory. A collection of artifacts that provide" , "the protobuf compiler and the GRPC plugin (if needed). Note that only" , "artifacts of the specified targets are considered (no runfiles etc.)." , "Specifying this field extends artifacts from \"base\"." ] , "deps": [ "Optional CC libraries the resulting CC proto libraries implicitly" , "depend on. Those are typically \"libprotobuf\" for CC proto libraries" , "and \"libgrpc++\" for CC proto service libraries. Specifying this" , "field extends dependencies from \"base\"." ] , "PROTOC": [ "The proto compiler. If \"toolchain\" is empty, this field's value is" , "considered the proto compiler name that is looked up in \"PATH\". If" , "\"toolchain\" is non-empty, this field's value is assumed to be the" , "relative path to the proto compiler in \"toolchain\". Specifying this" , "field overwrites values from \"base\"." ] , "GRPC_PLUGIN": [ "The GRPC plugin for the proto compiler. If \"toolchain\" is empty," , "this field's value is considered to be the absolute system path to the" , "plugin. If \"toolchain\" is non-empty, this field's value is assumed" , "to be the relative path to the plugin in \"toolchain\". Specifying" , "this field overwrites values from \"base\"." ] , "PROTOCFLAGS": [ "Protobuf compiler flags. Specifying this field overwrites values from" , "\"base\"." ] , "ADD_PROTOCFLAGS": [ "Additional protobuf compiler flags. Specifying this field extends" , "values from \"base\"." ] , "LDFLAGS": [ "Linker flags for linking the final CC library. Specifying this field" , "overwrites values from \"base\"." ] , "ADD_LDFLAGS": [ "Additional linker flags for linking the final CC library. Specifying" , "this field extends values from \"base\"." ] , "PATH": [ "Path for looking up the proto compiler. Individual paths are joined" , "with \":\". Specifying this field extends values from \"base\"." ] } , "config_doc": { "ARCH": [ "The unqualified architecture. Is taken as a default for \"HOST_ARCH\"" , "and \"TARGET_ARCH\" if not set." ] , "HOST_ARCH": ["The architecture on which the build actions are carried out."] , "DEBUG": [ "If logically true (typically, a non-empty map), use debug-related" , "options, otherwise not." ] } , "imports": { "base-provides": ["./", "..", "defaults-base-provides"] , "base-provides-++": ["./", "..", "defaults-base-provides-++"] , "base-provides-list": ["./", "..", "defaults-base-provides-list"] , "artifacts_list": ["", "field_artifacts_list"] , "compile-deps": ["CC", "compile-deps"] , "compile-args-deps": ["CC", "compile-args-deps"] , "link-deps": ["CC", "link-deps"] , "link-args-deps": ["CC", "link-args-deps"] , "cflags-files-deps": ["CC", "cflags-files-deps"] , "ldflags-files-deps": ["CC", "ldflags-files-deps"] , "for host": ["transitions", "for host"] , "debug-deps": ["CC", "debug-deps"] } , "config_transitions": {"toolchain": [{"type": "CALL_EXPRESSION", "name": "for host"}]} , "expression": { "type": "let*" , "bindings": [ ["PROTOC", {"type": "FIELD", "name": "PROTOC"}] , ["PROTOCFLAGS", {"type": "FIELD", "name": "PROTOCFLAGS"}] , ["LDFLAGS", {"type": "FIELD", "name": "LDFLAGS"}] , ["GRPC_PLUGIN", {"type": "FIELD", "name": "GRPC_PLUGIN"}] , ["PATH", {"type": "FIELD", "name": "PATH"}] , ["provider", "PROTOC"] , [ "PROTOC" , { "type": "if" , "cond": {"type": "var", "name": "PROTOC"} , "then": {"type": "var", "name": "PROTOC"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "PROTOCFLAGS"] , [ "PROTOCFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "PROTOCFLAGS"} , "then": {"type": "var", "name": "PROTOCFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "LDFLAGS"] , [ "LDFLAGS" , { "type": "if" , "cond": {"type": "var", "name": "LDFLAGS"} , "then": {"type": "var", "name": "LDFLAGS"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] , ["provider", "GRPC_PLUGIN"] , [ "GRPC_PLUGIN" , { "type": "if" , "cond": {"type": "var", "name": "GRPC_PLUGIN"} , "then": {"type": "var", "name": "GRPC_PLUGIN"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "PATH"] , [ "PATH" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "PATH"} , {"type": "CALL_EXPRESSION", "name": "base-provides-++"} ] } } ] , ["provider", "ENV"] , ["default", {"type": "empty_map"}] , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] , ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] , [ "NON_SYSTEM_TOOLS" , { "type": "map_union" , "$1": { "type": "++" , "$1": [ [{"type": "CALL_EXPRESSION", "name": "base-provides"}] , { "type": "if" , "cond": {"type": "FIELD", "name": "PROTOC"} , "then": [ { "type": "singleton_map" , "key": "PROTOC" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "if" , "cond": {"type": "FIELD", "name": "GRPC_PLUGIN"} , "then": [ { "type": "singleton_map" , "key": "GRPC_PLUGIN" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } ] } } ] , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] , [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "toolchain artifacts may not overlap" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} , { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": { "type": "let*" , "bindings": [ ["fieldname", "toolchain"] , [ "transition" , {"type": "CALL_EXPRESSION", "name": "for host"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } ] } } ] , [ "PROTOCFLAGS" , { "type": "++" , "$1": [ {"type": "var", "name": "PROTOCFLAGS"} , {"type": "FIELD", "name": "ADD_PROTOCFLAGS"} ] } ] , [ "LDFLAGS" , { "type": "++" , "$1": [ {"type": "var", "name": "LDFLAGS"} , {"type": "FIELD", "name": "ADD_LDFLAGS"} ] } ] , ["deps-fieldnames", ["base", "deps"]] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , ["link-args", {"type": "CALL_EXPRESSION", "name": "link-args-deps"}] , [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , ["package", {"type": "env", "vars": ["cflags-files", "ldflags-files"]}] , [ "debug-srcs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "let*" , "bindings": [["deps-provider", "debug-srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , "else": {"type": "empty_map"} } ] , [ "debug-hdrs" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": { "type": "map_union" , "$1": [ { "type": "let*" , "bindings": [["deps-provider", "debug-hdrs"]] , "body": {"type": "CALL_EXPRESSION", "name": "debug-deps"} } , {"type": "var", "name": "compile-deps"} ] } , "else": {"type": "empty_map"} } ] ] , "body": { "type": "RESULT" , "provides": { "type": "env" , "vars": [ "PROTOC" , "PROTOCFLAGS" , "LDFLAGS" , "GRPC_PLUGIN" , "PATH" , "ENV" , "TOOLCHAIN" , "NON_SYSTEM_TOOLS" , "compile-deps" , "compile-args" , "link-deps" , "link-args" , "package" , "debug-srcs" , "debug-hdrs" ] } } } } , "library": { "doc": [ "A C++ library, generated from proto files." , "" , "This rule usually is used to bind anonymous targets generated from" , "proto libraries." ] , "string_fields": ["name", "stage"] , "target_fields": ["srcs", "deps"] , "config_vars": ["CXX", "CXXFLAGS", "ADD_CXXFLAGS", "AR", "ENV", "DEBUG"] , "implicit": {"defaults": [["./", "..", "defaults"]], "proto-defaults": ["defaults"]} , "imports": {"protoc-compile": "protoc-compile"} , "expression": { "type": "let*" , "bindings": [ ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "DEBUG" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": {"type": "var", "name": "DEBUG"} , "else": {"type": "empty_map"} } ] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , ["public-fieldnames", ["deps", "proto-defaults"]] , ["private-fieldnames", ["deps", "proto-defaults"]] ] , "body": {"type": "CALL_EXPRESSION", "name": "protoc-compile"} } } , "service library": { "doc": [ "A service C++ library, generated from proto files." , "" , "Calls protoc with gRPC plugin to additionally generate gRPC services" , "from proto libraries." ] , "string_fields": ["name", "stage"] , "target_fields": ["srcs", "deps"] , "config_vars": ["CXX", "CXXFLAGS", "ADD_CXXFLAGS", "AR", "ENV", "DEBUG"] , "implicit": { "defaults": [["./", "..", "defaults"]] , "proto-defaults": ["service defaults"] } , "imports": {"protoc-compile": "protoc-compile"} , "expression": { "type": "let*" , "bindings": [ ["service support", true] , ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "DEBUG" , { "type": "if" , "cond": {"type": "var", "name": "DEBUG"} , "then": {"type": "var", "name": "DEBUG"} , "else": {"type": "empty_map"} } ] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , ["public-fieldnames", ["deps", "proto-defaults"]] , ["private-fieldnames", ["deps", "proto-defaults"]] ] , "body": {"type": "CALL_EXPRESSION", "name": "protoc-compile"} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/proto/TARGETS000066400000000000000000000007271516554100600234430ustar00rootroot00000000000000{ "defaults": { "type": ["CC/proto", "defaults"] , "PROTOC": ["protoc"] , "PATH": ["/bin", "/usr/bin"] , "deps": ["libprotobuf"] } , "service defaults": { "type": ["CC/proto", "defaults"] , "base": ["defaults"] , "GRPC_PLUGIN": ["/usr/bin/grpc_cpp_plugin"] , "deps": ["libgrpc++"] } , "libprotobuf": {"type": ["CC/pkgconfig", "system_library"], "name": ["protobuf"]} , "libgrpc++": {"type": ["CC/pkgconfig", "system_library"], "name": ["grpc++"]} } just-buildsystem-justbuild-b1fb5fa/rules/CC/test/000077500000000000000000000000001516554100600222155ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/CC/test/EXPRESSIONS000066400000000000000000000304161516554100600237660ustar00rootroot00000000000000{ "run_test": { "doc": [ "Build and run a CC test binary using the provided runner." , "" , "Note that if variable RUNS_PER_TEST contains a non-false value, a" , "summarizer must be provided." ] , "vars": [ "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "ARCH_DISPATCH" , "TEST_SUMMARY_EXECUTION_PROPERTIES" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "TEST_ENV" , "TIMEOUT_SCALE" , "CC_TEST_LAUNCHER" , "RUNS_PER_TEST" , "name" , "pure C" , "stage" , "srcs" , "private-hdrs" , "private-defines" , "private-cflags" , "private-ldflags" , "defaults-transition" , "deps-transition" , "deps-fieldnames" , "runner" , "runner-data" , "test-args" , "test-data" , "summarizer" , "summary artifacts" , "LINT" ] , "imports": { "artifacts": ["./", "../..", "field_artifacts"] , "runfiles": ["./", "../..", "field_runfiles"] , "compile-deps": ["./", "..", "compile-deps"] , "compile-args-deps": ["./", "..", "compile-args-deps"] , "link-deps": ["./", "..", "link-deps"] , "link-args-deps": ["./", "..", "link-args-deps"] , "cflags-files-deps": ["./", "..", "cflags-files-deps"] , "ldflags-files-deps": ["./", "..", "ldflags-files-deps"] , "binary": ["./", "..", "bin artifact"] , "host transition": ["transitions", "for host"] , "target properties": ["transitions", "target properties"] , "stage": ["./", "../..", "stage_singleton_field"] , "lint": ["./", "..", "lint information"] } , "expression": { "type": "let*" , "bindings": [ [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , { "type": "++" , "$1": [ { "type": "foreach" , "var": "def" , "range": {"type": "var", "name": "private-defines", "default": []} , "body": {"type": "join", "$1": ["-D", {"type": "var", "name": "def"}]} } , {"type": "var", "name": "private-cflags", "default": []} , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] } ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , [ "link-args" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "link-args-deps"} , {"type": "var", "name": "private-ldflags", "default": []} ] } ] , ["binary", {"type": "CALL_EXPRESSION", "name": "binary"}] , [ "lint" , { "type": "if" , "cond": {"type": "var", "name": "LINT"} , "then": { "type": "let*" , "bindings": [ ["hdrs", {"type": "empty_map"}] , [ "lint-deps fieldnames" , ["private-hdrs", "srcs", "private-deps"] ] ] , "body": {"type": "CALL_EXPRESSION", "name": "lint"} } } ] , [ "staged test binary" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "binary"} , "var_val": "binary" , "body": { "type": "singleton_map" , "key": "test" , "value": {"type": "var", "name": "binary"} } } } ] , [ "test-args" , { "type": "singleton_map" , "key": "test-args.json" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": {"type": "var", "name": "test-args", "default": []} } } } ] , [ "test-launcher" , { "type": "singleton_map" , "key": "test-launcher.json" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": {"type": "var", "name": "CC_TEST_LAUNCHER", "default": []} } } } ] , [ "test-name" , { "type": "join" , "separator": "/" , "$1": [{"type": "var", "name": "stage"}, {"type": "var", "name": "name"}] } ] , [ "test input" , { "type": "map_union" , "$1": [ { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "test-data"} } , {"type": "var", "name": "runner"} , { "type": "var" , "name": "runner-data" , "default": {"type": "empty_map"} } , {"type": "var", "name": "test-args"} , {"type": "var", "name": "test-launcher"} , {"type": "var", "name": "staged test binary"} ] } ] , [ "target properties" , {"type": "CALL_EXPRESSION", "name": "target properties"} ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "RUNS_PER_TEST"} , "else": { "type": "let*" , "bindings": [ [ "test-results" , { "type": "ACTION" , "outs": [ "result" , "stdout" , "stderr" , "time-start" , "time-stop" , "pwd" ] , "inputs": {"type": "var", "name": "test input"} , "cmd": ["./runner"] , "env": { "type": "var" , "name": "TEST_ENV" , "default": {"type": "empty_map"} } , "may_fail": ["test"] , "fail_message": { "type": "join" , "$1": ["CC test ", {"type": "var", "name": "test-name"}, " failed"] } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "execution properties": {"type": "var", "name": "target properties"} } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "test-results"} , "runfiles": {"type": "var", "name": "runfiles"} , "provides": {"type": "env", "vars": ["lint"]} } } , "then": { "type": "let*" , "bindings": [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "ATTEMPT" , "range": { "type": "range" , "$1": {"type": "var", "name": "RUNS_PER_TEST"} } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} , "value": { "type": "ACTION" , "outs": ["result", "stdout", "stderr", "time-start", "time-stop"] , "inputs": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "ATTEMPT" , "value": { "type": "BLOB" , "data": {"type": "var", "name": "ATTEMPT"} } } , {"type": "var", "name": "test input"} ] } , "cmd": ["./runner"] , "env": { "type": "var" , "name": "TEST_ENV" , "default": {"type": "empty_map"} } , "may_fail": ["test"] , "no_cache": ["test"] , "fail_message": { "type": "join" , "$1": [ "CC test " , {"type": "var", "name": "test-name"} , " failed (Run" , {"type": "var", "name": "ATTEMPT"} , ")" ] } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "execution properties": {"type": "var", "name": "target properties"} } } } } ] , [ "attempts (for summary)" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "summary artifacts"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "$_"} , "key": {"type": "var", "name": "_"} } } } } } } } } ] , [ "summary" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } , "outs": ["stdout", "stderr", "result", "time-start", "time-stop"] , "cmd": ["./summarizer"] , "execution properties": { "type": "var" , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" , "default": {"type": "empty_map"} } } ] , [ "attempts" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "$_"}} } } } ] , [ "artifacts" , { "type": "`" , "$1": { "pwd": {"type": ",", "$1": {"type": "BLOB", "data": "/summary"}} , "work": { "type": "," , "$1": {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} } } } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "artifacts"} , "runfiles": {"type": "var", "name": "runfiles"} , "provides": {"type": "env", "vars": ["lint"]} } } } } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/test/RULES000066400000000000000000000235411516554100600230370ustar00rootroot00000000000000{ "test": { "doc": ["A test written in C++"] , "tainted": ["test"] , "target_fields": ["srcs", "private-hdrs", "private-deps", "data"] , "string_fields": [ "name" , "args" , "stage" , "pure C" , "private-defines" , "private-cflags" , "private-ldflags" ] , "config_vars": [ "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "TEST_ENV" , "TIMEOUT_SCALE" , "CC_TEST_LAUNCHER" , "RUNS_PER_TEST" , "ARCH_DISPATCH" , "TEST_SUMMARY_EXECUTION_PROPERTIES" , "LINT" ] , "implicit": { "defaults": [["./", "..", "defaults"]] , "runner": ["runner"] , "summarizer": [["./", "../../shell/test", "summarizer"]] } , "field_doc": { "name": [ "The name of the test" , "" , "Used to name the test binary as well as for staging the test result" ] , "args": ["Command line arguments for the test binary"] , "srcs": [ "The sources of the test binary" , "" , "The resulting test binary is run in an environment where it can assume" , "that the environment variable TEST_TMPDIR points to a" , "directory that may be used exclusively by this test." , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." , "" , "This running of the test is carried out by the implicit dependency" , "on the target \"runner\". By setting this target in the target layer" , "of this rules repository (instead of letting it default to the" , "respective file), the C/C++ test environment can be modified globally." ] , "private-hdrs": [ "Any additional header files that need to be present when compiling" , "the test binary." ] , "private-defines": [ "List of defines set for source files local to this target." , "Each list entry will be prepended by \"-D\"." ] , "private-cflags": ["List of compile flags set for source files local to this target."] , "private-ldflags": [ "Additional linker flags for linking external libraries (not built" , "by this tool, typically system libraries)." ] , "stage": [ "The logical location of all header and source files." , "Individual directory components are joined with \"/\"." ] , "data": ["Any files the test binary needs access to when running"] , "defaults": ["The C/C++ toolchain to use"] , "runner": [ "The test runner which starts the actual test binary after providing" , "the respective environment. The runner also takes care of capturing" , "stdout/stderr and timing information." ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" , "detection) to an overall test result. If more fields than the result" , "itself is needed, those can be specified using the \"summarizer\" rule." ] } , "config_doc": { "CC": ["The name of the C compiler to be used."] , "CXX": ["The name of the C++ compiler to be used."] , "CFLAGS": [ "The flags for CC to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX" , "taken from the [\"CC\", \"defaults\"] target" ] , "ENV": ["The environment for any action generated."] , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": ["Factor on how to scale the timeout for this test. Defaults to 1.0."] , "CC_TEST_LAUNCHER": [ "List of strings representing the launcher that is prepend to the" , "command line for running the test binary." ] , "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." , "" , "Test runs are summarized by the [\"shell/test\", \"summarizer\"] that" , "is also used by shell tests." ] , "TARGET_ARCH": [ "The architecture to build the test for." , "" , "Will only be honored, if that architecture is available in the" , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" , "on the host architecture." ] , "ARCH_DISPATCH": [ "Map of architectures to execution properties that ensure execution" , "on that architecture. Only the actual test binary will be run with" , "the specified execution properties (i.e., on the target architecture);" , "all building will be done on the host architecture." ] , "TEST_SUMMARY_EXECUTION_PROPERTIES": [ "Additional remote-execution properties for the test-summarizing action" , "in case RUNS_PER_TEST is set; defaults to the empty map." ] , "LINT": [ "Also provide nodes describing compile actions and header files;" , "those can be used by lint rules (doing also the config transition)" , "for additional checks." ] } , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its" , "\"private-deps\" argument, this gives an easy way of defining test" , "suites." ] , "imports": { "artifacts": ["./", "../..", "field_artifacts"] , "runfiles": ["./", "../..", "field_runfiles"] , "host transition": ["transitions", "maybe for host"] , "stage": ["./", "../..", "stage_singleton_field"] , "run_test": "run_test" , "field_list": ["", "field_list_provider"] } , "config_transitions": { "defaults": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-hdrs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "srcs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "data": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ [ "name" , { "type": "assert_non_empty" , "msg": "A non-empty name has to be provided for binaries" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["pure C", {"type": "FIELD", "name": "pure C"}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , ["host-trans", {"type": "CALL_EXPRESSION", "name": "host transition"}] , [ "srcs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "srcs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "private-hdrs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , ["defaults-transition", {"type": "var", "name": "host-trans"}] , ["deps-transition", {"type": "var", "name": "host-trans"}] , ["deps-fieldnames", ["private-deps", "defaults"]] , [ "runner" , { "type": "let*" , "bindings": [ ["fieldname", "runner"] , ["location", "runner"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , ["test-args", {"type": "FIELD", "name": "args", "default": []}] , [ "test-data" , { "type": "let*" , "bindings": [ ["fieldname", "data"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "runfiles"} , {"type": "CALL_EXPRESSION", "name": "artifacts"} ] } } ] , [ "summarizer" , { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "summary artifacts" , { "type": "++" , "$1": [ ["result"] , { "type": "let*" , "bindings": [["provider", "artifacts"], ["fieldname", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} } ] } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "run_test"} } } } just-buildsystem-justbuild-b1fb5fa/rules/CC/test/TARGETS000066400000000000000000000000031516554100600232420ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/rules/CC/test/runner000077500000000000000000000037241516554100600234620ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import time time_start: float = time.time() time_stop: float = 0 result: str = "UNKNOWN" stderr: str = "" stdout: str = "" def dump_results() -> None: with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) with open("pwd", "w") as f: f.write("%s\n" % (os.getcwd(), )) dump_results() TEMP_DIR = os.path.realpath("scratch") os.makedirs(TEMP_DIR, exist_ok=True) WORK_DIR = os.path.realpath("work") os.makedirs(WORK_DIR, exist_ok=True) ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR, TMPDIR=TEMP_DIR) with open('test-launcher.json') as f: test_launcher = json.load(f) with open('test-args.json') as f: test_args = json.load(f) ret = subprocess.run(test_launcher + ["../test"] + test_args, cwd=WORK_DIR, env=ENV, capture_output=True) time_stop = time.time() result = "PASS" if ret.returncode == 0 else "FAIL" stdout = ret.stdout.decode("utf-8") stderr = ret.stderr.decode("utf-8") dump_results() if result != "PASS": exit(1) just-buildsystem-justbuild-b1fb5fa/rules/EXPRESSIONS000066400000000000000000000170421516554100600225220ustar00rootroot00000000000000{ "field_artifacts_list": { "doc": ["Query list of artifacts from target_field's targets"] , "vars": ["fieldname", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "transition": ["The optional configuration transition for the targets."] } , "expression": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "x"} , "transition": {"type": "var", "name": "transition", "default": {"type": "empty_map"}} } } } , "field_artifacts": { "doc": ["Query and merge artifacts from target_field's targets"] , "vars": ["fieldname", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "transition": ["The optional configuration transition for the targets."] } , "imports": {"artifacts_list": "field_artifacts_list"} , "expression": { "type": "disjoint_map_union" , "msg": ["artifacts", {"type": "var", "name": "fieldname"}, "must not overlap"] , "$1": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } , "field_runfiles_list": { "doc": ["Query list of runfiles from target_field's targets"] , "vars": ["fieldname", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "transition": ["The optional configuration transition for the targets."] } , "expression": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "DEP_RUNFILES" , "dep": {"type": "var", "name": "x"} , "transition": {"type": "var", "name": "transition", "default": {"type": "empty_map"}} } } } , "field_runfiles": { "doc": ["Query and merge runfiles from target_field's targets"] , "vars": ["fieldname", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "transition": ["The optional configuration transition for the targets."] } , "imports": {"runfiles_list": "field_runfiles_list"} , "expression": { "type": "disjoint_map_union" , "msg": ["runfiles", {"type": "var", "name": "fieldname"}, "must not overlap"] , "$1": {"type": "CALL_EXPRESSION", "name": "runfiles_list"} } } , "field_provider_list": { "doc": ["Query list of providers from targets' provides map"] , "vars": ["fieldname", "provider", "transition", "default"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "provider": ["The name of the map provider in the provides map."] , "transition": ["The optional configuration transition for the targets."] , "default": ["The default if the provider was not found (default: [])."] } , "expression": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "provider": {"type": "var", "name": "provider"} , "transition": {"type": "var", "name": "transition", "default": {"type": "empty_map"}} , "default": {"type": "var", "name": "default", "default": []} } } } , "field_map_provider": { "doc": ["Query and merge map-providers from targets' provides map"] , "vars": ["fieldname", "provider", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "provider": ["The name of the map provider in the provides map."] , "transition": ["The optional configuration transition for the targets."] } , "imports": {"provider_list": "field_provider_list"} , "expression": { "type": "disjoint_map_union" , "msg": ["Overlapping entries in provider", {"type": "var", "name": "provider"}] , "$1": { "type": "let*" , "bindings": [["default", {"type": "empty_map"}]] , "body": {"type": "CALL_EXPRESSION", "name": "provider_list"} } } } , "field_list_provider": { "doc": ["Query and merge list-providers from targets' provides map"] , "vars": ["fieldname", "provider", "transition"] , "vars_doc": { "fieldname": ["The name of the target_field to query."] , "provider": ["The name of the list provider in the provides map."] , "transition": ["The optional configuration transition for the targets."] } , "imports": {"provider_list": "field_provider_list"} , "expression": {"type": "++", "$1": {"type": "CALL_EXPRESSION", "name": "provider_list"}} } , "action_env": { "vars": ["ENV"] , "expression": { "type": "map_union" , "$1": [ {"type": "'", "$1": {"PATH": "/bin:/usr/bin"}} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } } , "stage_singleton_field": { "vars": ["fieldname", "transition", "location"] , "expression": { "type": "assert_non_empty" , "msg": ["No artifact specified in field", {"type": "var", "name": "fieldname"}] , "$1": { "type": "disjoint_map_union" , "msg": [ "Expecting (essentially) a single artifact in field" , {"type": "var", "name": "fieldname"} ] , "$1": { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "artifact" , "range": { "type": "values" , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "src"} , "transition": { "type": "var" , "name": "transition" , "default": {"type": "empty_map"} } } } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "location"} , "value": {"type": "var", "name": "artifact"} } } } } } } } , "stage_artifact_to_singleton_field": { "vars": ["artifact", "fieldname", "transition"] , "expression": { "type": "let*" , "bindings": [ [ "location" , { "type": "++" , "$1": { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": {"type": "var", "name": "fieldname"}} , "body": { "type": "keys" , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "src"} , "transition": { "type": "var" , "name": "transition" , "default": {"type": "empty_map"} } } } } } ] , [ "staged_artifact" , { "type": "foreach_map" , "range": {"type": "var", "name": "artifact"} , "var_val": "val" , "body": { "type": "foreach" , "range": {"type": "var", "name": "location"} , "var": "pos" , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "pos"} , "value": {"type": "var", "name": "val"} } } } ] ] , "body": { "type": "disjoint_map_union" , "$1": {"type": "++", "$1": {"type": "var", "name": "staged_artifact"}} } } } } just-buildsystem-justbuild-b1fb5fa/rules/data/000077500000000000000000000000001516554100600216625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/data/RULES000066400000000000000000000062751516554100600225110ustar00rootroot00000000000000{ "staged": { "doc": ["Stage data to a logical subdirectory."] , "target_fields": ["srcs", "deps"] , "string_fields": ["stage"] , "field_doc": { "srcs": ["The files to be staged"] , "stage": [ "The logical directory to stage the files to." , "Individual directory components are joined with \"/\"." ] , "deps": [ "Targets of with their runfiles should be added as well." , "Their staging is not changed." ] } , "artifacts_doc": [ "The runfiles of the \"srcs\" targets staged to the directory" , "specified in \"stage\" together the runfiles of the targets" , "specified in the field \"deps\" (in their original location)." ] , "runfiles_doc": ["Same as artifacts"] , "imports": { "runfiles": ["./", "..", "field_runfiles"] , "artifacts": ["./", "..", "field_artifacts"] } , "expression": { "type": "let*" , "bindings": [ [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "srcs" , { "type": "let*" , "bindings": [["fieldname", "srcs"]] , "body": { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "runfiles"} , {"type": "CALL_EXPRESSION", "name": "artifacts"} ] } } ] , [ "staged" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": {"type": "var", "name": "srcs"} } ] , [ "dep stage" , { "type": "let*" , "bindings": [["fieldname", "deps"]] , "body": {"type": "CALL_EXPRESSION", "name": "runfiles"} } ] , [ "total" , { "type": "disjoint_map_union" , "msg": "Conflict between staged data and dependencies" , "$1": [ {"type": "var", "name": "dep stage"} , {"type": "var", "name": "staged"} ] } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "total"} , "runfiles": {"type": "var", "name": "total"} } } } , "overlay": { "doc": ["Overlay the artifacts of \"deps\" targets in a latest-wins fashion."] , "target_fields": ["deps"] , "field_doc": {"deps": ["The targets of which to overlay the artifacts"]} , "artifacts_doc": [ "Artifacts of the targets specified in \"deps\". For conflicting" , "logical paths, the artifact is taken from the latest target (in" , "the \"deps\" field) that defines that logical path." ] , "runfiles_doc": ["Same as artifacts"] , "expression": { "type": "let*" , "bindings": [ [ "all" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "dep"}} } } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "all"} , "runfiles": {"type": "var", "name": "all"} } } } } just-buildsystem-justbuild-b1fb5fa/rules/lint/000077500000000000000000000000001516554100600217175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/lint/RULES000066400000000000000000000237101516554100600225370ustar00rootroot00000000000000{ "defaults": { "doc": [ "A rule to provide default paths common to all lint targets." , "" , "While lint targets bring their own host directory via the" , "field \"config\", it can still be necessary to bring some" , "default paths, e.g., for finding the compiler, or the interpreter" , "for the runner." ] , "string_fields": ["PATH"] , "target_fields": ["base"] , "field_doc": { "base": ["Other targets to inherit values from."] , "PATH": [ "Paths for looking up system tools." , "Specifying this field extends values from \"base\"." ] } , "imports": {"base-provides-++": ["CC", "defaults-base-provides-++"]} , "expression": { "type": "let*" , "bindings": [ [ "PATH" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "FIELD", "name": "PATH"} , { "type": "let*" , "bindings": [["provider", "PATH"]] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] } } ] ] , "body": {"type": "RESULT", "provides": {"type": "env", "vars": ["PATH"]}} } } , "targets": { "doc": [ "Run a given linter on the lint information provided by the given targets." ] , "target_fields": ["linter", "config", "summarizer", "targets"] , "string_fields": ["name"] , "tainted": ["lint"] , "field_doc": { "linter": [ "Single artifact running the lint checks." , "" , "This program is invoked with" , "- argv[1] the file to lint, and" , "- argv[2:] the original command line." , "This invocation happens in an environment with" , "- CONFIG pointing to the directory with all the artifacts given" , " by the field \"config\", and" , "- OUT pointing to a directory to which files with the lint result" , " can be written." , "- META pointing to a json file containing" , " - at key \"direct deps artifact names\" a list of all input" , " artifacts that come from the target itself or are runfiles of a" , " direct dependency." , " - at key \"extra outs\" a list of extra output artifacts that the" , " command might produce, such as DWARF objects if debug fission is" , " enabled." , "- TMPDIR pointing to a directory location that can be used to" , " create additional temporary files." , "It is supposed to indicate by the exit code whether the file to lint" , "complies with the given linting policy, with 0 meaning" , "compliant." , "Stdout and stderr, as well as the directory ${OUT} can be used to" , "provide additional information." ] , "config": ["Any configuration or other files needed by the linter."] , "summarizer": [ "Single artifact generating a summary of the individual lint results." , "It will be called in a directory where all subdirectories" , "except . and .. represent the results of the individual lint" , "actions. Those are given as" , " - a file \"result\" with content \"PASS\" if and only if the lint" , " action exited 0," , " - files \"stdout\" and \"stderr\" with stdout and stderr of the lint" , " action, and" , " - a directory \"out\" with the additional information provided by the" , " lint action." , "The summarizer is required to indicate the overall result by the exit" , "code, produce a human-readable summary on stdout, and optionally" , "additional information in the directory ${OUT}." ] , "call_lint": ["Launcher for the linter"] , "call_summary": ["Launcher for the summarizer"] , "name": [ "Name of the kind of lint check performed, to be reported" , "when an action is failing." ] } , "implicit": { "defaults": ["defaults"] , "call_lint": ["call_lint"] , "call_summary": ["call_summary"] } , "config_transitions": {"targets": [{"type": "'", "$1": {"LINT": true}}]} , "anonymous": {"lint": {"target": "targets", "provider": "lint", "rule_map": {}}} , "imports": { "stage": ["", "stage_singleton_field"] , "artifacts": ["", "field_artifacts"] , "default-PATH": ["CC", "default-PATH"] } , "expression": { "type": "let*" , "bindings": [ [ "linter" , { "type": "let*" , "bindings": [["fieldname", "linter"], ["location", "linter"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "runner" , { "type": "let*" , "bindings": [["fieldname", "call_lint"], ["location", "runner"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "config" , { "type": "let*" , "bindings": [["fieldname", "config"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } ] , [ "PATH" , { "type": "join" , "separator": ":" , "$1": {"type": "CALL_EXPRESSION", "name": "default-PATH"} } ] , ["name", {"type": "join", "$1": {"type": "FIELD", "name": "name"}}] , [ "lint results" , { "type": "foreach" , "range": {"type": "FIELD", "name": "lint"} , "body": { "type": "let*" , "bindings": [ [ "src" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "src" } ] , [ "cmd" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "cmd" } ] , [ "src input" , { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "_"} } ] , [ "direct deps artifact names" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "direct deps artifact names" } ] , [ "extra outs" , { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "extra outs" } ] ] , "body": { "type": "TREE" , "$1": { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "runner"} , {"type": "var", "name": "linter"} , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "src input"} } , { "type": "to_subdir" , "subdir": "config" , "$1": {"type": "var", "name": "config"} } , { "type": "singleton_map" , "key": "meta.json" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": { "type": "env" , "vars": ["direct deps artifact names", "extra outs"] } } } } ] } , "cmd": { "type": "++" , "$1": [ ["./runner", {"type": "var", "name": "src"}] , {"type": "var", "name": "cmd"} ] } , "env": {"type": "env", "vars": ["PATH"]} , "outs": ["stdout", "stderr", "result"] , "out_dirs": ["out"] , "may_fail": ["lint"] , "fail_message": { "type": "join" , "$1": [ "lint " , {"type": "var", "name": "name"} , " failed for " , {"type": "var", "name": "src"} ] } } } } } ] , [ "lint results" , {"type": "nub_right", "$1": {"type": "var", "name": "lint results"}} ] , [ "summary input" , {"type": "enumerate", "$1": {"type": "var", "name": "lint results"}} ] , [ "summarizer" , { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "runner" , { "type": "let*" , "bindings": [["fieldname", "call_summary"], ["location", "runner"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "summary" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "runner"} , {"type": "var", "name": "summarizer"} , { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "summary input"} } ] } , "cmd": ["./runner"] , "outs": ["report", "result"] , "out_dirs": ["out"] } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": [ {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": {"type": "TREE", "$1": {"type": "var", "name": "summary input"}} } ] } } } } } just-buildsystem-justbuild-b1fb5fa/rules/lint/TARGETS000066400000000000000000000001341516554100600227510ustar00rootroot00000000000000{ "defaults": {"type": "defaults", "base": [["CC", "defaults"], ["shell", "defaults"]]} } just-buildsystem-justbuild-b1fb5fa/rules/lint/call_lint000077500000000000000000000020541516554100600236070ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ensure all required outputs are present touch stdout touch stderr RESULT=UNKNOWN echo "${RESULT}" > result mkdir -p out export OUT="$(pwd)/out" mkdir -p config export CONFIG="$(pwd)/config" export META="$(pwd)/meta.json" mkdir scratch export TMPDIR=$(realpath scratch) cd work if ../linter "$@" > ../stdout 2> ../stderr then RESULT=PASS else RESULT=FAIL fi echo "${RESULT}" > ../result if [ "${RESULT}" '!=' PASS ] then exit 1; fi just-buildsystem-justbuild-b1fb5fa/rules/lint/call_summary000077500000000000000000000015531516554100600243410ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ensure all required outputs are present touch report RESULT=UNKNOWN echo "${RESULT}" > result mkdir -p out export OUT="$(pwd)/out" cd work if ../summarizer > ../report then RESULT=PASS else RESULT=FAIL fi echo "${RESULT}" > ../result exit 0 just-buildsystem-justbuild-b1fb5fa/rules/patch/000077500000000000000000000000001516554100600220505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/patch/EXPRESSIONS000066400000000000000000000043741516554100600236250ustar00rootroot00000000000000{ "default-PATCH": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "patch-defaults"] , ["provider", "PATCH"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-ENV": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "patch-defaults"] , ["provider", "ENV"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-PATH": { "vars": ["defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "patch-defaults"] , ["provider", "PATH"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } , "default-TOOLCHAIN": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "..", "field_map_provider"]} , "expression": { "type": "let*" , "bindings": [ ["fieldname", "patch-defaults"] , ["provider", "TOOLCHAIN"] , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } , "default-NON_SYSTEM_TOOLS": { "vars": ["defaults-transition"] , "expression": { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "patch-defaults"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "provider": "NON_SYSTEM_TOOLS" , "transition": { "type": "var" , "name": "defaults-transition" , "default": {"type": "empty_map"} } , "default": {"type": "empty_map"} } } } } } just-buildsystem-justbuild-b1fb5fa/rules/patch/RULES000066400000000000000000000266011516554100600226720ustar00rootroot00000000000000{ "defaults": { "doc": [ "A rule to provide defaults." , "All targets take their defaults for PATCH from the target" , "[\"patch\", \"defaults\"]. This is probably the only sensible" , "use of this rule. As targets form a different root, the defaults" , "can be provided without changing this directory." ] , "target_fields": ["base", "toolchain"] , "string_fields": ["PATCH", "PATH", "SYSTEM_TOOLS"] , "field_doc": { "base": ["Other targets (using the same rule) to inherit values from."] , "toolchain": [ "Optional toolchain directory. A collection of artifacts that provide" , "the tool PATCH. Note that only artifacts of" , "the specified targets are considered (no runfiles etc.). Specifying" , "this field extends artifacts from \"base\"." ] , "PATCH": ["The patch binary to use"] , "SYSTEM_TOOLS": [ "List of tools (\"PATCH\") that should be taken from" , "the system instead of from \"toolchain\" (if specified)." ] , "PATH": [ "Path for looking up the compilers. Individual paths are joined" , "with \":\". Specifying this field extends values from \"base\"." ] } , "config_vars": ["ARCH", "HOST_ARCH"] , "imports": { "base-provides": ["CC", "defaults-base-provides"] , "base-provides-++": ["CC", "defaults-base-provides-++"] , "base-provides-list": ["CC", "defaults-base-provides-list"] , "artifacts_list": ["", "field_artifacts_list"] , "for host": ["transitions", "for host"] } , "config_transitions": {"toolchain": [{"type": "CALL_EXPRESSION", "name": "for host"}]} , "expression": { "type": "let*" , "bindings": [ ["PATCH", {"type": "FIELD", "name": "PATCH"}] , ["PATH", {"type": "FIELD", "name": "PATH"}] , ["provider", "PATCH"] , [ "PATCH" , { "type": "if" , "cond": {"type": "var", "name": "PATCH"} , "then": {"type": "var", "name": "PATCH"} , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , ["provider", "PATH"] , [ "PATH" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "var", "name": "PATH"} , {"type": "CALL_EXPRESSION", "name": "base-provides-++"} ] } } ] , ["provider", "ENV"] , ["default", {"type": "empty_map"}] , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] , ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] , [ "NON_SYSTEM_TOOLS" , { "type": "map_union" , "$1": { "type": "++" , "$1": [ [{"type": "CALL_EXPRESSION", "name": "base-provides"}] , { "type": "if" , "cond": {"type": "FIELD", "name": "PATCH"} , "then": [ { "type": "singleton_map" , "key": "PATCH" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } , { "type": "foreach" , "range": {"type": "FIELD", "name": "SYSTEM_TOOLS"} , "var": "tool" , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "tool"} , "value": false } } ] } } ] , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] , [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "toolchain artifacts may not overlap" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} , { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": { "type": "let*" , "bindings": [ ["fieldname", "toolchain"] , [ "transition" , {"type": "CALL_EXPRESSION", "name": "for host"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } ] } } ] ] , "body": { "type": "RESULT" , "provides": { "type": "env" , "vars": ["PATCH", "PATH", "ENV", "TOOLCHAIN", "NON_SYSTEM_TOOLS"] } } } } , "file": { "doc": ["Replace a file, logically in place, by a patched version"] , "target_fields": ["src", "patch"] , "config_vars": ["PATCH", "ENV"] , "implicit": {"patch-defaults": ["defaults"], "defaults": [["shell", "defaults"]]} , "field_doc": { "src": [ "The single source file to patch, typically an explicit file reference." ] , "patch": ["The patch to apply."] , "patch-defaults": ["The patch binary (and toolchain) to use"] , "defaults": ["The shell toolchain to use"] } , "artifacts_doc": ["The patched file, staged to the position the of the original file"] , "runfiles_doc": ["Same as artifacts"] , "imports": { "stage_field": ["./", "..", "stage_singleton_field"] , "stage_artifact": ["./", "..", "stage_artifact_to_singleton_field"] , "default-PATCH": "default-PATCH" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-ENV": "default-ENV" , "default-PATH": "default-PATH" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" , "sh-TOOLCHAIN": ["CC", "default-TOOLCHAIN"] , "sh-PATH": ["CC", "default-PATH"] , "sh": ["shell", "sh"] , "list_provider": ["./", "..", "field_list_provider"] } , "expression": { "type": "let*" , "bindings": [ ["TOOLCHAIN_DIR", "toolchain"] , [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "Staging conflict between patch and sh toolchain" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"} , {"type": "CALL_EXPRESSION", "name": "sh-TOOLCHAIN"} ] } ] , [ "TOOLCHAIN" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} , "$1": {"type": "var", "name": "TOOLCHAIN"} } ] , [ "bin dirs" , { "type": "let*" , "bindings": [["fieldname", "defaults"], ["provider", "bin dirs"]] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } ] , [ "bin dirs" , { "type": "foreach" , "range": {"type": "var", "name": "bin dirs"} , "body": { "type": "join" , "$1": [ "./" , {"type": "var", "name": "TOOLCHAIN_DIR"} , "/" , {"type": "var", "name": "_"} ] } } ] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , [ "PATCH" , { "type": "var" , "name": "PATCH" , "default": { "type": "join" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": { "type": "lookup" , "key": "PATCH" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] } , {"type": "CALL_EXPRESSION", "name": "default-PATCH"} ] } } } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} ] } ] , [ "ENV_PATH" , { "type": "lookup" , "map": {"type": "var", "name": "ENV"} , "key": "PATH" } ] , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "var", "name": "ENV"} , { "type": "singleton_map" , "key": "PATH" , "value": { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} , {"type": "CALL_EXPRESSION", "name": "sh-PATH"} , {"type": "var", "name": "bin dirs"} , { "type": "if" , "cond": {"type": "var", "name": "ENV_PATH"} , "then": [{"type": "var", "name": "ENV_PATH"}] } ] } } } ] } ] , ["sh", {"type": "CALL_EXPRESSION", "name": "sh"}] , [ "orig" , { "type": "let*" , "bindings": [["fieldname", "src"], ["location", "orig"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage_field"} } ] , [ "patch" , { "type": "let*" , "bindings": [["fieldname", "patch"], ["location", "patch"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage_field"} } ] , [ "script" , { "type": "singleton_map" , "key": "run_patch.sh" , "value": { "type": "BLOB" , "data": { "type": "join" , "separator": "\n" , "$1": [ "set -e" , "cp orig patched" , "chmod +w patched" , { "type": "join" , "$1": [ { "type": "join_cmd" , "$1": [ {"type": "var", "name": "PATCH", "default": "patch"} , "patched" , "patch" ] } , " >log || (cat log && exit 1)" ] } ] } } } ] , [ "inputs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "orig"} , {"type": "var", "name": "patch"} , {"type": "var", "name": "TOOLCHAIN"} , {"type": "var", "name": "script"} ] } ] , [ "patched" , { "type": "ACTION" , "inputs": {"type": "var", "name": "inputs"} , "outs": ["patched"] , "cmd": [{"type": "var", "name": "sh"}, "./run_patch.sh"] , "env": {"type": "var", "name": "ENV"} } ] , [ "result" , { "type": "let*" , "bindings": [ ["artifact", {"type": "var", "name": "patched"}] , ["fieldname", "src"] ] , "body": {"type": "CALL_EXPRESSION", "name": "stage_artifact"} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "result"} , "runfiles": {"type": "var", "name": "result"} } } } } just-buildsystem-justbuild-b1fb5fa/rules/patch/TARGETS000066400000000000000000000001561516554100600231060ustar00rootroot00000000000000{ "defaults": { "type": ["patch", "defaults"] , "PATCH": ["patch"] , "PATH": ["/bin", "/usr/bin"] } } just-buildsystem-justbuild-b1fb5fa/rules/proto/000077500000000000000000000000001516554100600221145ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/proto/RULES000066400000000000000000000062351516554100600227370ustar00rootroot00000000000000{ "library": { "doc": [ "A proto library as abstract data structure." , "" , "Such a library does not produce any artifacts itself, but it can be" , "used as a dependency for various language-specific rules." ] , "target_fields": ["srcs", "deps"] , "string_fields": ["stage", "name", "service"] , "field_doc": { "srcs": ["The proto files for this library"] , "deps": ["Any other proto library this library depends on"] , "stage": [ "The directory to stage the source files to." , "Directory components are joined by \"/\"." ] , "name": ["The name of the (abstract) library."] , "service": [ "If non empty, generate a service library (with access to \"rpc\"" , "definitions) instead of a regular one." ] } , "artifacts_doc": ["None"] , "runfiles_doc": ["None"] , "provides_doc": { "proto": [ "A list containing a single target-graph node with the definition of" , "this proto library. The node types generated are \"library\" and" , "\"service library\"." ] } , "imports": { "artifacts_list": ["./", "..", "field_artifacts_list"] , "list_provider": ["./", "..", "field_list_provider"] } , "expression": { "type": "let*" , "bindings": [ [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "name" , { "type": "assert_non_empty" , "msg": "Have to provide a name, unique in the stage" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , [ "srcs" , [ { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "disjoint_map_union" , "msg": "Sources have to be conflict free" , "$1": { "type": "let*" , "bindings": [["fieldname", "srcs"]] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } } } } } ] ] , [ "deps" , { "type": "let*" , "bindings": [["fieldname", "deps"], ["provider", "proto"]] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } ] , ["name", [{"type": "var", "name": "name"}]] , ["stage", [{"type": "var", "name": "stage"}]] , [ "proto" , [ { "type": "ABSTRACT_NODE" , "node_type": { "type": "if" , "cond": {"type": "FIELD", "name": "service"} , "then": "service library" , "else": "library" } , "target_fields": {"type": "env", "vars": ["srcs", "deps"]} , "string_fields": {"type": "env", "vars": ["name", "stage"]} } ] ] ] , "body": {"type": "RESULT", "provides": {"type": "env", "vars": ["proto"]}} } } } just-buildsystem-justbuild-b1fb5fa/rules/shell/000077500000000000000000000000001516554100600220605ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/shell/EXPRESSIONS000066400000000000000000000055571516554100600236410ustar00rootroot00000000000000{ "default-sh": { "vars": ["defaults-transition", "fieldname"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ [ "fieldname" , {"type": "var", "name": "fieldname", "default": "defaults"} ] , ["provider", "sh"] , ["transition", {"type": "var", "name": "defaults-transition"}] , [ "provided sh" , { "type": "join" , "$1": {"type": "CALL_EXPRESSION", "name": "list_provider"} } ] , [ "sh" , { "type": "if" , "cond": {"type": "var", "name": "provided sh"} , "then": {"type": "var", "name": "provided sh"} , "else": "sh" } ] ] , "body": {"type": "var", "name": "sh"} } } , "NON_SYSTEM": { "vars": ["defaults-transition", "fieldname"] , "expression": { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": { "type": "FIELD" , "name": {"type": "var", "name": "fieldname", "default": "defaults"} } , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "provider": "NON_SYSTEM_TOOLS" , "transition": { "type": "var" , "name": "defaults-transition" , "default": {"type": "empty_map"} } , "default": {"type": "empty_map"} } } } } , "sh": { "vars": ["defaults-transition", "TOOLCHAIN_DIR", "fieldname"] , "imports": {"default-sh": "default-sh", "NON_SYSTEM": "NON_SYSTEM"} , "expression": { "type": "let*" , "bindings": [ ["sh", {"type": "CALL_EXPRESSION", "name": "default-sh"}] , ["NON_SYSTEM", {"type": "CALL_EXPRESSION", "name": "NON_SYSTEM"}] , [ "sh" , { "type": "if" , "cond": { "type": "lookup" , "key": "sh" , "map": {"type": "var", "name": "NON_SYSTEM"} } , "then": { "type": "join" , "separator": "/" , "$1": [ "." , { "type": "var" , "name": "TOOLCHAIN_DIR" , "default": "toolchain" } , {"type": "var", "name": "sh"} ] } , "else": {"type": "var", "name": "sh"} } ] ] , "body": {"type": "var", "name": "sh"} } } , "PATH": { "vars": ["fieldname", "defaults-transition"] , "imports": {"list_provider": ["./", "..", "field_list_provider"]} , "expression": { "type": "let*" , "bindings": [ [ "fieldname" , {"type": "var", "name": "fieldname", "default": "defaults"} ] , ["provider", "PATH"] , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } } just-buildsystem-justbuild-b1fb5fa/rules/shell/RULES000066400000000000000000000120471516554100600227010ustar00rootroot00000000000000{ "defaults": { "doc": [ "A rule to provide defaults for the usage of the shell" , "" , "All targets using invocations of the shell use the target" , "[\"shell\", \"defaults\"] to determine which shell to use and how to" , "invoke it. The definition of this default target is probably the only" , "meaningful use of this rule." ] , "target_fields": ["base", "toolchain"] , "string_fields": ["sh", "PATH", "bin dirs"] , "field_doc": { "base": ["Other targets (using the same rule) to inherit values from."] , "toolchain": [ "Optional toolchain directory. A collection of artifacts that" , "form the toolchain, in particular the shell itself, where not taken" , "from the ambient host environment." , "Values provided from base are extended." , "This field is built for the host." ] , "sh": [ "The name of the sh binary; if the the field \"toolchain\" is" , "not empty, the value is interpreted as relative to the toolchain" , "directory." ] , "PATH": [ "Paths for looking up system tools." , "Specifying this field extends values from \"base\"." ] , "bin dirs": [ "Directories of the toolchain that contain additional binaries." , "Shell-specific rules will add those into PATH." ] } , "config_vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH"] , "imports": { "for host": ["transitions", "for host"] , "artifacts_list": ["", "field_artifacts_list"] , "base-provides-list": ["CC", "defaults-base-provides-list"] , "base-provides-++": ["CC", "defaults-base-provides-++"] , "base-provides": ["CC", "defaults-base-provides"] } , "config_transitions": {"toolchain": [{"type": "CALL_EXPRESSION", "name": "for host"}]} , "expression": { "type": "let*" , "bindings": [ [ "TOOLCHAIN" , { "type": "disjoint_map_union" , "msg": "toolchain artifacts must not overlap" , "$1": { "type": "++" , "$1": [ { "type": "let*" , "bindings": [ ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} } , { "type": "let*" , "bindings": [ ["fieldname", "toolchain"] , [ "transition" , {"type": "CALL_EXPRESSION", "name": "for host"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts_list"} } ] } } ] , [ "sh" , { "type": "if" , "cond": {"type": "FIELD", "name": "sh"} , "then": {"type": "FIELD", "name": "sh"} , "else": { "type": "let*" , "bindings": [["provider", "sh"]] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides"} } } ] , [ "NON_SYSTEM_TOOLS" , { "type": "map_union" , "$1": { "type": "++" , "$1": [ [ { "type": "let*" , "bindings": [ ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] , { "type": "if" , "cond": {"type": "FIELD", "name": "sh"} , "then": [ { "type": "singleton_map" , "key": "sh" , "value": { "type": "if" , "cond": {"type": "FIELD", "name": "toolchain"} , "then": true , "else": false } } ] } ] } } ] , [ "PATH" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "FIELD", "name": "PATH"} , { "type": "let*" , "bindings": [["provider", "PATH"]] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] } } ] , [ "bin dirs" , { "type": "nub_left" , "$1": { "type": "++" , "$1": [ {"type": "FIELD", "name": "bin dirs"} , { "type": "let*" , "bindings": [["provider", "bin dirs"]] , "body": {"type": "CALL_EXPRESSION", "name": "base-provides-++"} } ] } } ] ] , "body": { "type": "RESULT" , "provides": { "type": "env" , "vars": ["TOOLCHAIN", "sh", "NON_SYSTEM_TOOLS", "PATH", "bin dirs"] } } } } } just-buildsystem-justbuild-b1fb5fa/rules/shell/TARGETS000066400000000000000000000000431516554100600231110ustar00rootroot00000000000000{"defaults": {"type": "defaults"}} just-buildsystem-justbuild-b1fb5fa/rules/shell/test/000077500000000000000000000000001516554100600230375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/shell/test/EXPRESSIONS000066400000000000000000000215451516554100600246130ustar00rootroot00000000000000{ "test-action": { "vars": [ "TEST_ENV" , "TIMEOUT_SCALE" , "ATTEMPT" , "name" , "test.sh" , "keep" , "keep-dirs" , "runner" , "deps-fieldname" , "deps-transition" , "target properties" ] , "imports": { "artifacts_list": ["./", "../..", "field_artifacts_list"] , "runfiles_list": ["./", "../..", "field_runfiles_list"] , "default-TOOLCHAIN": ["./", "../../CC", "default-TOOLCHAIN"] , "default-NON_SYSTEM_TOOLS": ["./", "../../CC", "default-NON_SYSTEM_TOOLS"] , "default-PATH": ["./", "../../CC", "default-PATH"] , "default-sh": ["./", "..", "default-sh"] } , "expression": { "type": "let*" , "bindings": [ [ "runner" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "runner" , "range": {"type": "var", "name": "runner"} , "body": { "type": "map_union" , "$1": { "type": "foreach" , "var": "runner" , "range": { "type": "values" , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "runner"} } } , "body": {"type": "env", "vars": ["runner"]} } } } } ] , ["toolchain dirname", "toolchain"] , [ "toolchain" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "toolchain dirname"} , "$1": {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"} } ] , ["sh", {"type": "CALL_EXPRESSION", "name": "default-sh"}] , [ "NON_SYSTEM_TOOLS" , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} ] , [ "sh from workdir" , { "type": "if" , "cond": { "type": "lookup" , "key": "sh" , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} } , "then": { "type": "join" , "$1": ["../toolchain/", {"type": "var", "name": "sh"}] } , "else": {"type": "var", "name": "sh"} } ] , [ "invocation cmd" , [{"type": "var", "name": "sh from workdir"}, "../test.sh"] ] , [ "invocation" , { "type": "singleton_map" , "key": "invocation" , "value": { "type": "BLOB" , "data": { "type": "join_cmd" , "$1": {"type": "var", "name": "invocation cmd"} } } } ] , [ "test_env" , {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} ] , [ "test_env PATH" , { "type": "lookup" , "key": "PATH" , "map": {"type": "var", "name": "test_env"} } ] , [ "PATH" , { "type": "join" , "separator": ":" , "$1": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "test_env PATH"} , "then": [{"type": "var", "name": "test_env PATH"}] } , {"type": "CALL_EXPRESSION", "name": "default-PATH"} ] } } ] , [ "test_env" , { "type": "if" , "cond": {"type": "var", "name": "PATH"} , "then": { "type": "map_union" , "$1": [ {"type": "var", "name": "test_env"} , { "type": "singleton_map" , "key": "PATH" , "value": {"type": "var", "name": "PATH"} } ] } , "else": {"type": "var", "name": "test_env"} } ] , [ "deps" , { "type": "TREE" , "$1": { "type": "disjoint_map_union" , "msg": [ "Field" , {"type": "var", "name": "deps-fieldname"} , "has to stage in a conflict free way" ] , "$1": { "type": "++" , "$1": { "type": "let*" , "bindings": [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": [ {"type": "CALL_EXPRESSION", "name": "runfiles_list"} , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} ] } } } } ] , [ "attempt marker" , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "ATTEMPT"} , "$2": null } , "then": {"type": "empty_map"} , "else": { "type": "singleton_map" , "key": "ATTEMPT" , "value": {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}} } } ] , [ "outs" , { "type": "++" , "$1": [ ["result", "stdout", "stderr", "time-start", "time-stop", "pwd"] , { "type": "foreach" , "var": "filename" , "range": {"type": "var", "name": "keep"} , "body": { "type": "join" , "$1": ["work/", {"type": "var", "name": "filename"}] } } ] } ] , [ "out_dirs" , { "type": "foreach" , "var": "dir_path" , "range": {"type": "var", "name": "keep-dirs"} , "body": { "type": "join" , "$1": ["work/", {"type": "var", "name": "dir_path"}] } } ] , [ "inputs" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "work" , "value": {"type": "var", "name": "deps"} } , {"type": "var", "name": "toolchain"} , {"type": "var", "name": "runner"} , {"type": "var", "name": "invocation"} , {"type": "var", "name": "test.sh"} , {"type": "var", "name": "attempt marker"} ] } ] , [ "cmd" , {"type": "++", "$1": [["./runner"], {"type": "var", "name": "keep"}]} ] ] , "body": { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null} , "then": { "type": "ACTION" , "outs": {"type": "var", "name": "outs"} , "out_dirs": {"type": "var", "name": "out_dirs"} , "inputs": {"type": "var", "name": "inputs"} , "cmd": {"type": "var", "name": "cmd"} , "env": {"type": "var", "name": "test_env"} , "may_fail": ["test"] , "fail_message": { "type": "join" , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"] } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "execution properties": {"type": "var", "name": "target properties"} } , "else": { "type": "ACTION" , "outs": {"type": "var", "name": "outs"} , "out_dirs": {"type": "var", "name": "out_dirs"} , "inputs": {"type": "var", "name": "inputs"} , "cmd": {"type": "var", "name": "cmd"} , "env": {"type": "var", "name": "test_env"} , "may_fail": ["test"] , "no_cache": ["test"] , "fail_message": { "type": "join" , "$1": [ "shell test " , {"type": "var", "name": "name"} , " failed (Run " , {"type": "var", "name": "ATTEMPT"} , ")" ] } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "execution properties": {"type": "var", "name": "target properties"} } } } } , "test-result": { "vars": [ "TEST_ENV" , "TIMEOUT_SCALE" , "name" , "test.sh" , "keep" , "keep-dirs" , "runner" , "deps-fieldname" , "deps-transition" , "target properties" , "lint" ] , "imports": {"action": "test-action"} , "expression": { "type": "let*" , "bindings": [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "test-results"} , "runfiles": {"type": "var", "name": "runfiles"} , "provides": {"type": "env", "vars": ["lint"]} } } } } just-buildsystem-justbuild-b1fb5fa/rules/shell/test/RULES000066400000000000000000000311771516554100600236650ustar00rootroot00000000000000{ "summarizer": { "doc": ["Specify a test summarizer together with the required additional fields"] , "target_fields": ["summarizer"] , "string_fields": ["artifacts"] , "imports": {"stage": ["./", "../..", "stage_singleton_field"]} , "field_doc": { "summarizer": ["The single artifact acting as summarizer"] , "artifacts": [ "Any additional artifacts, besides \"result\", the summaries needs from" , "the individual test results" ] } , "expression": { "type": "RESULT" , "artifacts": { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } , "provides": { "type": "singleton_map" , "key": "artifacts" , "value": {"type": "FIELD", "name": "artifacts"} } } } , "script": { "doc": ["Shell test, given by a test script"] , "target_fields": ["deps", "test"] , "string_fields": ["keep", "keep-dirs", "name"] , "config_vars": [ "ARCH" , "HOST_ARCH" , "RUNS_PER_TEST" , "TEST_ENV" , "TIMEOUT_SCALE" , "TARGET_ARCH" , "ARCH_DISPATCH" , "TEST_SUMMARY_EXECUTION_PROPERTIES" , "LINT" ] , "field_doc": { "test": [ "The shell script for the test, launched with sh." , "" , "An empty directory is created to store any temporary files needed" , "by the test, and it is made available in the environment variable" , "TEST_TMPDIR. The test should not assume write permissions" , "outside the working directory and the TEST_TMPDIR." , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." , "" , "This running of the test is carried out by the implicit dependency" , "on the target \"runner\". By setting this target in the target layer" , "of this rues repository (instead of letting it default to the" , "respective file), the shell test environment can be modified globally." ] , "name": [ "A name for the test, used in reporting, as well as for staging" , "the test result tree in the runfiles" ] , "keep": [ "List of names (relative to the test working directory) of files that" , "the test might generate that should be kept as part of the output." , "This might be useful for further analysis of the test" ] , "keep-dirs": [ "List of names (relative to the test working directory) of directories" , "that the test might generate that should be kept as part of the" , "output. This might be useful for further analysis of the test" ] , "deps": [ "Any targets that should be staged (with artifacts and runfiles) into" , "the tests working directory" ] , "runner": [ "The test runner which starts the actual test script after providing" , "the respective environment. The runner also takes care of capturing" , "stdout/stderr, timing information, and ensure the presence of the" , "files to keep even if the script failed to produce them." ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" , "detection) to an overall test result. If more fields than the result" , "itself is needed, those can be specified using the \"summarizer\" rule." ] , "defaults": ["The shell toolcahin to use."] } , "config_doc": { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." , "" , "The individual test runs will be summarized by the implicit dependency" , "on the target \"summarizer\". By setting this target in the target" , "in the target layer of this rues repository (instead of letting it" , "default to the respective file) the layout of the summary can be" , "changed globally." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": ["Factor on how to scale the timeout for this test. Defaults to 1.0."] , "TARGET_ARCH": [ "The architecture to build the test for." , "" , "Will only be honored, if that architecture is available in the" , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" , "on the host architecture." ] , "ARCH_DISPATCH": [ "Map of architectures to execution properties that ensure execution" , "on that architecture. Only the actual test binary will be run with" , "the specified execution properties (i.e., on the target architecture);" , "all building will be done on the host architecture." ] , "TEST_SUMMARY_EXECUTION_PROPERTIES": [ "Additional remote-execution properties for the test-summarizing action" , "in case RUNS_PER_TEST is set; defaults to the empty map." ] } , "tainted": ["test"] , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "work: In this directory, all the files specified to \"keep\" and" , " \"keep-dirs\" are staged" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] , "implicit": { "runner": ["runner"] , "summarizer": ["summarizer"] , "defaults": [["./", "..", "defaults"]] } , "imports": { "test-result": "test-result" , "action": "test-action" , "stage": ["./", "../..", "stage_singleton_field"] , "host transition": ["transitions", "maybe for host"] , "target properties": ["transitions", "target properties"] , "field_list": ["", "field_list_provider"] } , "config_transitions": { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ [ "test.sh" , { "type": "context" , "msg": "Expecting 'test' to specify precisely one file containing a shell script" , "$1": { "type": "let*" , "bindings": [ ["fieldname", "test"] , ["location", "test.sh"] , [ "transition" , {"type": "CALL_EXPRESSION", "name": "host transition"} ] ] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } } ] , [ "name" , { "type": "assert_non_empty" , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["keep", {"type": "FIELD", "name": "keep"}] , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}] , ["runner", {"type": "FIELD", "name": "runner"}] , ["deps-fieldname", "deps"] , [ "deps-transition" , {"type": "CALL_EXPRESSION", "name": "host transition"} ] , [ "target properties" , {"type": "CALL_EXPRESSION", "name": "target properties"} ] , [ "lint" , { "type": "if" , "cond": {"type": "var", "name": "LINT"} , "then": { "type": "let*" , "bindings": [ ["fieldname", "deps"] , ["provider", "lint"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} } } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "RUNS_PER_TEST"} , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} , "then": { "type": "let*" , "bindings": [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "ATTEMPT" , "range": { "type": "range" , "$1": {"type": "var", "name": "RUNS_PER_TEST"} } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} , "value": {"type": "CALL_EXPRESSION", "name": "action"} } } } ] , [ "summarizer" , { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "summary artifacts" , { "type": "++" , "$1": [ ["result"] , { "type": "let*" , "bindings": [["provider", "artifacts"], ["fieldname", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} } ] } ] , [ "attempts (for summary)" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "summary artifacts"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "$_"} , "key": {"type": "var", "name": "_"} } } } } } } } } ] , [ "summary" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } , "outs": ["stdout", "stderr", "result", "time-start", "time-stop"] , "cmd": ["./summarizer"] , "execution properties": { "type": "var" , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" , "default": {"type": "empty_map"} } } ] , [ "attempts" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "$_"}} } } } ] , [ "artifacts" , { "type": "`" , "$1": { "pwd": {"type": ",", "$1": {"type": "BLOB", "data": "/summary"}} , "work": { "type": "," , "$1": {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} } } } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "artifacts"} , "runfiles": {"type": "var", "name": "runfiles"} , "provides": {"type": "env", "vars": ["lint"]} } } } } } } just-buildsystem-justbuild-b1fb5fa/rules/shell/test/TARGETS000066400000000000000000000002151516554100600240710ustar00rootroot00000000000000{ "summarizer": { "type": "summarizer" , "summarizer": [["FILE", null, "summarizer"]] , "artifacts": ["time-start", "time-stop"] } } just-buildsystem-justbuild-b1fb5fa/rules/shell/test/runner000077500000000000000000000027401516554100600243010ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ensure all required outputs are present touch stdout touch stderr RESULT=UNKNOWN echo "${RESULT}" > result echo UNKNOWN > time-start echo UNKNOWN > time-stop pwd > pwd export HOME="$(pwd)/DO-NO-USE-HOME" mkdir scratch export TEST_TMPDIR=$(realpath scratch) export TMPDIR="${TEST_TMPDIR}" # Change to the working directory; note: while unlikely, the test # might not have test data, so we have to ensure the presence of # the work directory. mkdir -p work cd work date +%s > ../time-start # TODO: # - proper wrapping with timeout if . ../invocation > ../stdout 2> ../stderr then RESULT=PASS else RESULT=FAIL fi date +%s > ../time-stop # Ensure all the promissed output files in the work directory # are present, even if the test failed to create them. for f in "$@" do touch "./${f}" done echo "${RESULT}" > ../result if [ "${RESULT}" '!=' PASS ] then exit 1; fi just-buildsystem-justbuild-b1fb5fa/rules/shell/test/summarizer000077500000000000000000000052451516554100600251710ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time from typing import Any, Dict, List g_RESULTS: Dict[str, List[Any]] = {} g_COUNT: float = 0 PASS_count: float = 0 PASS_time: float = 0 time_start: float = time.time() time_stop: float = 0 for attempt in os.listdir("."): if os.path.isdir(attempt): g_COUNT += 1 with open(os.path.join(attempt, "result")) as f: result = f.read().strip() g_RESULTS[result] = g_RESULTS.get(result, []) + [int(attempt)] try: with open(os.path.join(attempt, "time-start")) as f: start = float(f.read().strip()) time_start = min(time_start, start) except: pass try: with open(os.path.join(attempt, "time-stop")) as f: stop = float(f.read().strip()) time_stop = max(time_start, stop) except: pass if (start > 0) and (stop >= start) and result == "PASS": PASS_count += 1 PASS_time += stop - start result: str = "UNKNOWN" if set(g_RESULTS.keys()) <= set(["PASS", "FAIL"]): if not g_RESULTS.get("FAIL"): result = "PASS" elif not g_RESULTS.get("PASS"): result = "FAIL" else: result = "FLAKY" with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("Summary: %s\n\n" % (result, )) f.write("PASS: %s\n" % (sorted(g_RESULTS.get("PASS", [])), )) failures =sorted(g_RESULTS.get("FAIL", [])) f.write("FAIL: %s\n" % (failures, )) g_RESULTS.pop("PASS", None) g_RESULTS.pop("FAIL", None) if g_RESULTS: f.write("\nother results: %r\n" % (g_RESULTS, )) if result == "FLAKY": f.write("\nFailure rate %5.2f%%\n" % (100.0 * len(failures) / g_COUNT)) if PASS_count >= 2: f.write("\nAverage time of a passed test instance: %.1fs\n" % (PASS_time / PASS_count)) with open("stderr", "w") as f: pass just-buildsystem-justbuild-b1fb5fa/rules/test/000077500000000000000000000000001516554100600217305ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/test/EXPRESSIONS000066400000000000000000000032071516554100600234770ustar00rootroot00000000000000{ "matrix": { "vars": ["TEST_MATRIX"] , "expression": { "type": "foldl" , "var": "VAR" , "accum_var": "so far" , "range": { "type": "keys" , "$1": { "type": "var" , "name": "TEST_MATRIX" , "default": {"type": "empty_map"} } } , "start": [{"type": "'", "$1": {"": {"TEST_MATRIX": null}}}] , "body": { "type": "++" , "$1": { "type": "foreach_map" , "var_key": "stage" , "var_val": "VALUE" , "range": { "type": "lookup" , "key": {"type": "var", "name": "VAR"} , "map": {"type": "var", "name": "TEST_MATRIX"} } , "body": { "type": "++" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "so far"} , "body": { "type": "foreach_map" , "range": { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": {"type": "var", "name": "_"} } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "map_union" , "$1": [ {"type": "var", "name": "$_"} , { "type": "singleton_map" , "key": {"type": "var", "name": "VAR"} , "value": {"type": "var", "name": "VALUE"} } ] } } } } } } } } } } just-buildsystem-justbuild-b1fb5fa/rules/test/RULES000066400000000000000000000135731516554100600225560ustar00rootroot00000000000000{ "suite": { "doc": [ "Form a compound target out of many test targets." , "" , "More precisely, take the runfiles of the given \"deps\", take their" , "disjoint union and stage the result. Also propagate relevant" , "providers." ] , "tainted": ["test"] , "target_fields": ["deps"] , "string_fields": ["stage"] , "field_doc": { "deps": ["The targets that suite is composed of."] , "stage": [ "The logical location this test suite is to be placed." , "Individual entries will be joined with \"/\"." ] } , "artifacts_doc": [ "The disjoint union of the runfiles of the \"deps\" targets" , "staged as the location given by \"stage\"." ] , "runfiles_doc": ["Same as artifacts."] , "imports": { "runfiles": ["", "field_runfiles"] , "list_provider": ["", "field_list_provider"] } , "expression": { "type": "let*" , "bindings": [ ["fieldname", "deps"] , ["runfiles", {"type": "CALL_EXPRESSION", "name": "runfiles"}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "staged results" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": {"type": "var", "name": "runfiles"} } ] , ["provider", "lint"] , ["lint", {"type": "CALL_EXPRESSION", "name": "list_provider"}] , ["lint", {"type": "nub_right", "$1": {"type": "var", "name": "lint"}}] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "staged results"} , "runfiles": {"type": "var", "name": "staged results"} , "provides": {"type": "env", "vars": ["lint"]} } } } , "matrix": { "doc": [ "Given a group of tests, build them in a variety of configurations." , "" , "The configuration variable TEST_MATRIX is expected to be a map with" , "each value being a map itself. Sequentially for each key, all possible" , "values of the associated map are tried and staged to the appropriate" , "key. Thus, the tests in \"deps\" are built in an exponential number of" , "configurations." , "" , "If TEST_MATRIX is unset, {} will be assumed, i.e., all the \"deps\" will" , "be built precisely once, in the current configuration. In this way," , "the \"matrix\" rule can be used instead of the \"suite\" rule to allow" , "user-defined configuration matrices, dispatching over parameters for" , "dependencies (e.g., the toolchain)." ] , "tainted": ["test"] , "config_vars": ["TEST_MATRIX"] , "target_fields": ["deps"] , "string_fields": ["stage"] , "field_doc": { "deps": ["The targets that suite is composed of."] , "stage": [ "The logical location this test suite is to be placed." , "Individual entries will be joined with \"/\"." ] } , "config_doc": { "TEST_MATRIX": [ "Map describing the dimensions of the matrix to run the tests for." , "" , "Keys are config variables. The value for each key has to be a map" , "mapping the stage name to the corresponding value for that config" , "variable." ] } , "artifacts_doc": [ "The disjoint union of the runfiles of the \"deps\" targets," , "evaluated and staged as requested by TEST_MATRIX and finally" , "staged to the location given by \"stage\"." ] , "runfiles_doc": ["Same as artifacts."] , "imports": { "runfiles": ["", "field_runfiles"] , "list_provider": ["", "field_list_provider"] , "matrix": "matrix" } , "config_transitions": { "deps": { "type": "++" , "$1": { "type": "foreach" , "range": {"type": "CALL_EXPRESSION", "name": "matrix"} , "body": {"type": "values", "$1": {"type": "var", "name": "_"}} } } } , "expression": { "type": "let*" , "bindings": [ ["matrix", {"type": "CALL_EXPRESSION", "name": "matrix"}] , ["fieldname", "deps"] , [ "staged runfiles list" , { "type": "foreach" , "range": {"type": "var", "name": "matrix"} , "body": { "type": "foreach_map" , "range": {"type": "var", "name": "_"} , "var_val": "transition" , "body": { "type": "to_subdir" , "subdir": {"type": "var", "name": "_"} , "$1": {"type": "CALL_EXPRESSION", "name": "runfiles"} } } } ] , [ "staged matrix runfiles" , { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": {"type": "var", "name": "staged runfiles list"} } } ] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , [ "staged results" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": {"type": "var", "name": "staged matrix runfiles"} } ] , ["provider", "lint"] , [ "lint" , { "type": "++" , "$1": { "type": "++" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "matrix"} , "body": { "type": "foreach_map" , "range": {"type": "var", "name": "_"} , "var_val": "transition" , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } } } } ] , ["lint", {"type": "nub_right", "$1": {"type": "var", "name": "lint"}}] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "staged results"} , "runfiles": {"type": "var", "name": "staged results"} , "provides": {"type": "env", "vars": ["lint"]} } } } } just-buildsystem-justbuild-b1fb5fa/rules/transitions/000077500000000000000000000000001516554100600233265ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/rules/transitions/EXPRESSIONS000066400000000000000000000032721516554100600250770ustar00rootroot00000000000000{ "for host": { "vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH"] , "expression": { "type": "let*" , "bindings": [ [ "BUILD_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "HOST_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["BUILD_ARCH", "TARGET_ARCH"]} } } , "target properties": { "vars": ["ARCH", "TARGET_ARCH", "ARCH_DISPATCH"] , "expression": { "type": "let*" , "bindings": [ [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "TARGET_ARCH"} , "$2": null } , "then": {"type": "empty_map"} , "else": { "type": "lookup" , "map": { "type": "var" , "name": "ARCH_DISPATCH" , "default": {"type": "empty_map"} } , "key": {"type": "var", "name": "TARGET_ARCH"} , "default": {"type": "empty_map"} } } } } , "maybe for host": { "vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH", "ARCH_DISPATCH"] , "imports": {"target properties": "target properties", "for host": "for host"} , "expression": { "type": "if" , "cond": {"type": "CALL_EXPRESSION", "name": "target properties"} , "then": {"type": "empty_map"} , "else": {"type": "CALL_EXPRESSION", "name": "for host"} } } } just-buildsystem-justbuild-b1fb5fa/share/000077500000000000000000000000001516554100600207215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/share/just_complete.bash000066400000000000000000000157731516554100600244520ustar00rootroot00000000000000# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################### just completion _just_subcommand_options(){ local cmd=$1 for w in $(just $cmd --help) do [[ $w =~ ^-. ]] && printf "%s\n" ${w//,/" "} done } _just_targets(){ command -v python3 &>/dev/null || return python3 - </dev/null)) COMPREPLY=($(compgen -f -W "${_opts[*]} ${_targets[*]}" -- $word )) compopt -o plusdirs -o bashdefault -o default else COMPREPLY=($(compgen -W "${SUBCOMMANDS[*]}" -- $word)) fi } complete -F _just_completion just ########################### just-mr completion _just-mr_options(){ local cmd=$1 for w in $($cmd --help 2>/dev/null) do [[ $w =~ ^-. ]] && printf "%s\n" ${w//,/" "} done } _just-mr_parse_subcommand() { local readonly FLAGS=("--help\n-h\n--norc\ndo") # treat 'do' as flag local readonly OPTIONS=("--distdir\n--just\n--local-build-root\n--main\n--rc\n-C\n-L") shift while [ -n "$1" ]; do if echo -e "$FLAGS" | grep -q -- "^$1$"; then shift; continue; fi if echo -e "$OPTIONS" | grep -q -- "^$1$"; then shift; shift; continue; fi if [ "$1" = "--" ]; then shift; fi break done echo "$1" } _just-mr_repos(){ command -v python3 &>/dev/null || return local CONF=$(just-mr setup --all 2>/dev/null) if [ ! -f "$CONF" ]; then return; fi python3 - </dev/null) _just_completion else # just-mr top-level options local _opts=($(_just-mr_options "just-mr")) COMPREPLY=($(compgen -W "${_opts[*]} ${SUBCOMMANDS[*]}" -- $word)) fi } complete -F _just-mr_completion just-mr just-buildsystem-justbuild-b1fb5fa/share/man/000077500000000000000000000000001516554100600214745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/share/man/just-deduplicate-repos.1.md000066400000000000000000000071531516554100600265570ustar00rootroot00000000000000% JUST-DEDUPLICATE-REPOS(1) | General Commands Manual NAME ==== just-deduplicate-repos - remove duplicate repositories from a multi-repository configuration SYNOPSIS ======== **`just-deduplicate-repos`** \[*`repository name`*\]... DESCRIPTION =========== Read a multi-repository configuration from stdin and print to stdout a repository configuration with indistinguishable repositories merged to a single repository. In doing so, keep the `"main"` repository, as well as any repositories specified as arguments on the command line, even if that means that some indistinguishable repositories cannot be merged into one (as both have to be kept). RATIONALE ========= As **`just`**(1) is a multi-repository build system, typically imported dependencies also contain multi-repository set up. Hence, a project typically has three components to describe the involved logical repositories. First, there is a description of the local repositories, i.e., the logical repositories (as `"file"` repositories) that reside in this physical repository in the sense of the version-control system (typically **`git`**(1)). They are described in a file often called `etc/repos.template.json` in **`just-mr-repository-config`**(5) format with open names for the direct dependencies. Next, there is a description of where to get the direct dependencies and which branches to follow there. This description is typically a script piping said `repos.template.json` through a sequence of invocations of **`just-import-git`**(1) which also adds the indirect dependencies. Finally, the output of that script is the multi-repository configuration `etc/repos.json` for this project. This file, while being generated, is still committed into the repository. First of all, it contains additional information: the precise pinned versions of (all) the dependencies which might change over time. So committing this file allows others to build with the precise same dependencies (including for old versions of the project). Moreover, having the multi-repository configuration materialized in the repository allows offline builds once the dependencies have been fetched (possibly by a different project) to **`just-mr`**(1)'s local build root (possibly by the setup for a different project that happens to have those dependencies as well). To update the dependencies, the import script can be run again and the newly created `etc/repos.json` committed in an update commit (after verifying that this update does not break anything). The final reason to commit a generated `etc/repos.json` is to close the loop and allow this project to be the dependency of another one; to keep the work done by **`just-import-git`**(1) manageable it requires the imported repository to have all its (direct or indirect) dependencies described precisely in their multi-repository config. When a project has more than one direct dependency, it can happen that two of the direct dependencies have a common dependency. Simply chaining **`just-import-git`**(1) this dependency will end up twice in the final repository configuration. While this will not result in additional actions, it will increase the cost of the analysis. Moreover, not merging indistinguishable repositories will make the resulting `etc/repos.json` bigger and propagate those redundant copies to other projects importing this one; that mechanism can, over a long chain of imports, lead to exponential many of those redundant copies. To avoid this, **`just-deduplicate-repos`** can be added as a last step (before the JSON pretty printing) in the import script. See also ======== **`just-mr-repository-config`**(5), **`just-import-git`**(1) **`just-mr`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-graph-file.5.md000066400000000000000000000170001516554100600251600ustar00rootroot00000000000000% JUST GRAPH FILE(5) | File Formats Manual NAME ==== just-graph-file - The format of the action graph used by **`just`**(1) DESCRIPTION =========== The file is read as JSON. Any other serialization describing the same JSON object is equivalent. We assume, that in JSON objects, each key occurs at most once; it is implementation defined how repetitions of the same key are treated. Artifacts and their serialization --------------------------------- There are five different kind of artifacts. The serialization of the artifact is always a JSON object with two keys: *`"type"`* and *`"data"`*. The value for *`"type"`* is always on of the strings *`"LOCAL"`*, *`"KNOWN"`*, *`"ACTION"`*, *`"TREE"`*, or *`"TREE_OVERLAY"`*. The value for *`"data"`* is a JSON object with keys depending on which type the artifact is of. - *`"LOCAL"`* artifacts refer to source files in a logical repository. The describing data are - the *`"repository"`*, given as its global name, and - the *`"path"`*, given as path relative to the root of that repository. - *`"KNOWN"`* artifacts are described by the hash of their content. The describing data are - the *`"file_type"`*, which is a one-letter string, - *`"f"`* for regular, non-executable files, - *`"x"`* for executable files, or - *`"t"`* for trees. - the *`"id"`* specifying the (applicable) hash of the file given as its hexadecimal encoding, a string, and - the *`"size"`* of the artifact, a number, in bytes. - *`"ACTION"`* artifacts are the outputs of actions. The defining data are - the *`"id"`*, a string with the name of the action, and - the *`"path"`*, specifying the path of this output artifact, relative to the root of the action directory - *`"TREE"`* artifacts refer to trees defined in the action graph. The defining data are - the *`"id"`*, a string with the name of the tree. - *`"TREE_OVERLAY"`* artifacts refer to trees defined as the result of overlaying earlier trees. The defining data are - the *`"id"`*, a string with the name of the tree overlay. Actions and their serialization ------------------------------- Actions are given by the data described in the following sections. For each item a key is associated and the serialization of the action is a JSON object mapping those keys to the respective serialization of the values. For some data items default values are given; if the value for the respective item equals the default, the respective key-value pair may be omitted in the serialization. - *`"command"`* specifies the command to be executed. It is a non-empty list of strings that forms the argument vector; the first entry is taken as program. This key is mandatory. - *`"env"`* specifies the environment variables the command should be executed with. It is given as map from strings to strings. The default is the empty map. - *`"input"`* describes the files available to the action. The action must not modify them in any way. They are specified as map from paths to artifacts (the latter serialized as described). Paths have to be relative paths and are taken relative to the working directory of the action. The default is the empty map. - *`"output"`* describes the files the action is supposed to generate, if any. It is given as a list of strings, each specifying a file name by a path relative to the working directory of the action. The default is the empty list. However, every action has to produce some form of output, so if *`"output"`* is empty, *`"output_dirs"`* cannot be empty. - *`"output_dirs"`* describes the directory output of the action, if any. It is given as a list of strings, each specifying the a directory name by a path relative to the working directory of the action. The *`"output_dirs"`* may also specify directories from which individual files are specified as *`"output"`*. The default value for *`"output_dirs"`* is the empty list. However, every action to produce some form of output, so if *`"output_dirs"`* is empty, *`"output"`* cannot be empty. - *`"may_fail"`* can either be *`null`* or a string. If it is a string, the build should be continued, even if that action returns a non-zero exit state. If the action returns a non-zero exit code, that string should be shown to the user in a suitable way (e.g., printing on the console). Otherwise (i.e., if no *`"may_fail"`* string is given), the build should be aborted if the action returns a non-zero exit code. The default for *`"may_fail"`* is *`null`*, i.e., abort on non-zero exit code. - *`"no_cache"`* is a boolean. If *`true`*, the action should never be cached, not even on success. The default is *`false`*. The graph format ---------------- The action graph is given by a JSON object. - The value for the key *`"blobs"`* is a list of strings. Those strings should be considered known; they might (additionally to what was agreed ahead of time as known) referred to as *`"KNOWN"`* artifacts. - The value for the key *`"trees"`* is a JSON object, mapping the names of trees to their definition. The definition of a tree is JSON object mapping paths to artifacts (serialized in the way described earlier). The paths are always interpreted as relative paths, relative to the root of the tree. It is not a requirement that a new tree is defined for every subdirectory; if a path contains the hierarchy separator, which is slash, then implicitly a subdirectory is present, and all path going through that subdirectory contribute to its content. It is, however, an error, if the a path is a prefix of another one (with the comparison done in the canonical form of that path). - The value for the key *`"actions"`* is a JSON object, mapping names of actions to their respective definition (serialized as JSON). - The optional value (with default `{}`) for the key *`"tree_overlays"`* is a JSON object mapping the names of tree overlay to their definition. The definition is a JSON object with the following keys. - For the key *`"trees"`* a JSON array of the artifacts to be overlayed in the order they are to be overlayed (with latest-wins semantics for each path). - For the key *`"disjoint"`* a boolean indicating if it is to be considered an error of a conflict arises on any path of the trees to be overlayed. Additional keys --------------- Any JSON object described here might have additional keys on top of the ones described. Implementations reading the graph have to accept and ignore those. Implementations writing action-graph files should be aware that a future version of this file format might give a specific meaning to those extra keys. Graphs written by **`just`**(1) have the additional key *`"origins"`* in each action. The value is a list of all places where this action was requested (so often, but not always, the list has length 1). Each such place is described by a JSON object with the following keys. - *`"target"`* The target in which the action was requested. It is given as a list, either a full qualified named target given as *`"@"`* followed by global repository name, module, and target name, or an anonymous target, given by *`"#"`* followed by a hash of the rule binding and the node name. - *`"subtask"`* The running number, starting from 0, of the action, as given by the (deterministic) evaluation order of he defining expression for the rule that defined the target. - *`"config"`* The effective configuration for that target, a JSON object. See also ======== **`just`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-import-git.1.md000066400000000000000000000103661516554100600252410ustar00rootroot00000000000000% JUST-IMPORT-GIT(1) | General Commands Manual NAME ==== just-import-git - import one Git repository to a multi-repository configuration SYNOPSIS ======== **`just-import-git`** \[*`OPTION`*\]... *`URL`* \[*`foreign repository name`*\] DESCRIPTION =========== Extend an existing **`just-mr-repository-config`**(5) by adding one Git repository. In doing so, the dependencies declared in the imported repository are added as well and *`"file"`* repositories are transformed to *`"subdir"`* parts of the imported repository. This solves the problem, that a repository can refer to itself only as *`"."`* in a portable way. The importing party, however, always knows the URL it is importing from. The imported repositories are renamed in a way that no conflicts with already present repositories arise. The repositories pulled in as dependencies are named in a way to remind for which repositories they came as a dependency. This renaming is taken into account at all places repositories are referred to, i.e., the *`"bindings"`* field, as well as roots defined by reference to other repositories. Only the main parts of repositories are imported (*`"repository"`*, *`"bindings"`*, names, and roots). The *`"pragma"`* part, as well as unknown fields are dropped. The repository to import is specified by its URL. The resulting configuration is printed on standard output. OPTIONS ======= **`--as`** *`NAME`* Specify the name the imported repository should have in the final configuration. If not specified, default to the name the repository has in the configuration file of the repository imported. In any case, the name is amended if it conflicts with an already existing name. **`-b`** *`BRANCH`* The branch in the imported repository to use; this branch is also recorded as the branch to follow. Defaults to *`"master"`*. **`-C`** *`CONFIGFILE`* Use the specified file as the configuration to import into. The string *`-`* is treated as a request to take the config from stdin; so a file called *`-`* has to be described as *`.`*/*`-`*, or similar. If not specified, a config file is searched for in the same way as **`just-mr`** does when invoked with **`--norc`**. **`-h`**, **`--help`** Output a usage message and exit. **`--map`** *`THEIRS`* *`OURS`* Map repositories from the imported configuration to already existing ones. Those repositories are not imported (and the search for their transitive dependency is omitted) and instead the specified already existing repository is used. This is useful, if a sufficiently compatible repository already exists in the configuration. **`-R`** *`RELPATH`* Use the file, specified by path relative to the repository root, as multi-repository specification in the imported repository. If not specified, for a config file is searched in the same way as **`just-mr`** does, when invoked with **`--no-rc`**, however leaving out searches relative to global roots (*`"home"`* and *`"system"`*). In other words, *`repos.json`* and *`etc/repos.json`* are tried if this option is not given. **`--plain`** Pretend the foreign multi-repository specification is the canonical one for a single repository. Useful, if the repository to be imported does not have a repository configuration or should be imported without dependencies. **`--absent`** Add the pragma `{"absent": true}` to all imported repositories. **`--mirror`** *`URL`* Provides an alternative fetch location for the imported repository. Specifying this option multiple times will accumulate URLs in the order they appear on the command line. These URLs will not be used during the import, but instead will be recorded as the value of the `"mirrors"` key in the resulting configuration of the imported repository. See **`just-mr-repository-config`**(5). **`--inherit-env`** *`VAR`* Specify, for the imported repository, environment variables to be inherited when fetching the repository by calling `git`. These variables will not be taken into account during the import (where the whole environment is inherited), but instead will be recorded as the value for the `"inherit env"` key in the resulting configuration of the imported repository. See **`just-mr-repository-config`**(5). See also ======== **`git`**(1), **`just-deduplicate-repos`**(1), **`just-mr-repository-config`**(5), **`just-mr`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-lock-config.5.md000066400000000000000000000350201516554100600253370ustar00rootroot00000000000000% JUST-LOCK CONFIG(5) | File Formats Manual NAME ==== just-lock-config - The format of the input configuration used by **`just-lock`**(1) DESCRIPTION =========== In order for the **`just-lock`**(1) tool to generate a repository configuration file usable by the **`just-mr`**(1) launcher, it requires its own configuration file describing how that resulting configuration should be obtained. The file is read as JSON. Any other serialization describing the same JSON object is equivalent. It is assumed that in JSON objects keys occur at most once; it is implementation defined how repetitions of the same key are treated. Repository import description objects ------------------------------------- One of the main functionalities of the **`just-lock`**(1) tool is to import dependencies from other Just projects, as described in their repositories configuration file. From each such project, one or more repositories can be imported (with their respective transitive dependencies). A *repository import description* is a JSON object describing one such repository to be imported. The following fields are supported: - *`"repo"`* has a string value defining the name of the repository to be imported. This entry is optional. - *`"alias"`* has a string value defining the new name under which the imported repository should be found in the output configuration. This entry is optional. If not provided, the value for the key `"repo"` ***must*** exist and its value is used instead. - *`"map"`* has as value a JSON object with string key-value pairs defining a mapping between repository names that would be brought in by this import (possibly transitively) and already imported repositories. This mapping can be used to avoid additional duplications of repositories from multiple imports. This entry is optional. - *`"pragma"`* has as value a JSON object. This entry is optional. If the object contains the `{"absent": true}` object, it informs that the imported repository and all transitive repositories imported as a consequence should have the `{"absent": true}` pragma added to their description. If the object contains the `{"to_git": true}` object, it informs that all `"file"`-type repositories brought in as a consequence of this import should have the `{"to_git": true}` pragma added to their description. Source objects -------------- A *source* provides information about an operation that the **`just-lock`**(1) tool can perform in order to extend an initial repository description stub and obtain the output repository configuration. In most cases, this operation involves importing repositories from other Just projects, but a more general operation exists as well. Sources are given as JSON objects for which the string value to the mandatory key *`"source"`* defines a supported type. Each source type informs which other fields are available. Currently, the supported source types are *`"git"`*, *`"file"`*, *`"archive"`*, *`"git tree"`*, and *`"generic"`*. ### *`"git"`* It defines an import operation of one or more dependencies from a Just project under Git version control. The following fields are supported: - *`"source"`* defines the current *source* type. This entry is mandatory. - *`"repos"`* has as value a JSON list where each entry is a *repository import description*. This entry is mandatory. An empty list is treated as if the current *source* object is missing. - *`"url"`* has a string value defining the URL of the Git repository. This entry is mandatory. - *`"branch"`* has a string value providing the name of the branch to consider from the Git repository. This entry is mandatory. - *`"commit"`* has a string value providing the hash of the commit to be used. This entry is optional. If provided, it has to be specified in hex encoding and it must be a commit in the branch specified by the corresponding field. If not provided, the `HEAD` commit of that branch is used. - *`"mirrors"`* has as value a JSON list of strings defining alternative locations for the Git repository to be used if the given URL fails to provide it. This entry is optional. - *`"inherit env"`* has as value a JSON list which will be recorded as the value for the `"inherit env"` key in the output configuration for all imported `"git"`-type repositories (see **`just-mr-repository-config`**(5)). This entry is optional. - *`"as plain"`* has a boolean value. If the field evaluates to `true`, it informs **`just-lock`**(1) to consider the foreign repository configuration to be the canonical one for a single repository. This can be useful if the Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, if a pragma object with key `"special"` is provided, it will unconditionally be forwarded to the `"pragma"` object of the repository being imported for this source. This entry is optional. - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not evaluate to `true`, **`just-lock`**(1) will search for a configuration file in the same locations as **`just-mr`**(1) does when invoked with **`--norc`** in the root directory of the Git repository. ### *`"file"`* It defines an import operation of one or more dependencies from a Just project present as a local checkout. The following fields are supported: - *`"source"`* defines the current *source* type. This entry is mandatory. - *`"repos"`* has as value a JSON list where each entry is a *repository import description*. This entry is mandatory. An empty list is treated as if the current *source* object is missing. - *`"path"`* has a string value defining the path to the local checkout. This entry is mandatory. - *`"as plain"`* has a boolean value. If the field evaluates to `true`, it informs **`just-lock`**(1) to consider the foreign repository configuration to be the canonical one for a single repository. This can be useful if the Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, if a pragma object with key `"special"` is provided, it will unconditionally be forwarded to the `"pragma"` object of the repository being imported for this source. This entry is optional. - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not evaluate to `true`, **`just-lock`**(1) will search for a configuration file in the same locations as **`just-mr`**(1) does when invoked with **`--norc`** in the root directory of the Git repository. ### *`"archive"`* It defines an import operation of one or more dependencies from an archived Just project. The following fields are supported: - *`"source"`* defines the current *source* type. This entry is mandatory. - *`"repos"`* has as value a JSON list where each entry is a *repository import description*. This entry is mandatory. An empty list is treated as if the current *source* object is missing. - *`"fetch"`* has a string value defining the URL of the archive. This entry is mandatory. - *`"type"`* has a string value providing a supported archive type. This entry is mandatory. Available values are: - `"tar"`: a tarball archive - `"zip"`: a zip-like archive - *`"content"`* has a string value providing the hash of the archive. This entry is optional. If provided, it has to be specified in hex encoding and it must be the Git blob identifier of the archive content. If not provided, it is computed based on the fetched archive. - *`"subdir"`* has a string value providing the relative path to the sources root inside the archive. This entry is optional. If missing, the root directory of the unpacked archive is used. - *`"sha256"`* - *`"sha512"`* have string values providing respective checksums for the archive. These entries are optional. If provided, they have to be specified in hex encoding and they will be checked for the fetched archive. - *`"mirrors"`* has as value a JSON list of strings defining alternative locations for the archive to be used if the given URL fails to provide it. This entry is optional. - *`"as plain"`* has a boolean value. If the field evaluates to `true`, it informs **`just-lock`**(1) to consider the foreign repository configuration to be the canonical one for a single repository. This can be useful if the archived repository does not have a configuration file or should be imported as-is, without dependencies. This entry is optional. - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, if a pragma object with key `"special"` is provided, it will unconditionally be forwarded to the `"pragma"` object of the repository being imported for this source. This entry is optional. - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the unpacked archive root. This entry is optional. If not provided and the `"as plain"` field does not evaluate to `true`, **`just-lock`**(1) will search for a configuration file in the same locations as **`just-mr`**(1) does when invoked with **`--norc`** in the root directory of the unpacked archive. ### *`"git tree"`* It defines an import operation of one or more dependencies from a Just project given as the result of running a command. This can be used, for example, to import projects found under a version control system other than Git. The following fields are supported: - *`"source"`* defines the current *source* type. This entry is mandatory. - *`"repos"`* has as value a JSON list where each entry is a *repository import description*. This entry is mandatory. An empty list is treated as if the current *source* object is missing. - *`"cmd"`* provides a list of strings forming a command that, when executed in an empty directory (anywhere in the file system), creates the tree of the source Just project to use for the import. This entry is optional. One and only one of the fields `"cmd"` and `"cmd gen"` must be provided. - *`"cmd gen"`* provides a list of strings forming a command that, when executed in an empty directory (anywhere in the file system), prints to stdout a string giving a JSON serialization of a valid input for the field `"cmd"` to be used. This entry is optional. One and only one of the fields `"cmd"` and `"cmd gen"` must be provided. - *`"env"`* provides a map of envariables to be set for executing the command and the command generator, if given. This entry is optional. - *`"inherit env"`* provides a list of variables to be inherited from the environment `just-lock` is called within, if set there. This entry is optional. - *`"subdir"`* has a string value providing the relative path to the sources root inside the generated tree. This entry is optional. If missing, the root directory of the generated tree is considered. - *`"as plain"`* has a boolean value. If the field evaluates to `true`, it informs **`just-lock`**(1) to consider the foreign repository configuration to be the canonical one for a single repository. This can be useful if the Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, if a pragma object with key `"special"` is provided, it will unconditionally be forwarded to the `"pragma"` object of the repository being imported for this source. This entry is optional. - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not evaluate to `true`, **`just-lock`**(1) will search for a configuration file in the same locations as **`just-mr`**(1) does when invoked with **`--norc`** in the root directory of the Git repository. ### *`"generic"`* It defines a liberal way to modify a `just-mr` configuration via a user-provided command. The following fields are supported: - *`"source"`* defines the current *source* type. This entry is mandatory. - *`"cmd"`* provides a list of strings forming a command that accepts a string as input from `stdin` containing a serialized `just-mr` configuration and outputs a string to `stdout` containing a serialized `just-mr` configuration. This entry is mandatory. - *`cwd`* provides the path to the directory in which the command will be run. If it is relative, it will be taken relative to the current working directory. This entry is optional. If missing, `"."` is used. - *`"env"`* provides a map of envariables to be set for executing the command. This entry is optional. - *`"inherit env"`* provides a list of variables to be inherited from the environment `just-lock` is called within, if set there. This entry is optional. The just-lock configuration format ---------------------------------- The configuration format is structured as a JSON object. It is a superset of the **`just-mr-repository-config`**(5), which is extended by two additional fields. Specifically, the following fields are supported: - *`"main"`* has the syntax and semantics as described in **`just-mr-repository-config`**(5). - *`"repositories"`* has the syntax and semantics as described in **`just-mr-repository-config`**(5). - *`"imports"`* is a JSON list with each entry a *source* object. - *`"keep"`* is a JSON list of strings defining the global names of repositories to be kept, together with the `"main"` repository, in the output configuration during the deduplication step of **`just-lock`**(1). Additional keys --------------- Any JSON object described in this format might have additional keys besides the ones mentioned. The current strategy of **`just-lock`**(1) is to accept and ignore them. Users should be aware that future versions of this format might give specific meanings to these extra keys. See also ======== **`just-lock`**(1), **`just-mr`**(1), **`just-mr-repository-config`**(5) just-buildsystem-justbuild-b1fb5fa/share/man/just-lock.1.md000066400000000000000000000071431516554100600240750ustar00rootroot00000000000000% JUST-LOCK(1) | General Commands Manual NAME ==== just-lock - generate and maintain a multi-repository configuration file SYNOPSIS ======== **`just-lock`** \[*`OPTION`*\]... DESCRIPTION =========== Just-Lock is a tool that encompasses several functionalities related to generating and maintaining the **`just-mr-repository-config`**(5) of a Just project, centered around multi-repository configuration composition. The main functionality of the tool is to import declared dependencies from other Just projects and generate a repository configuration which can directly be used by **`just-mr`**(1). The imported repositories are renamed in a way that no conflicts arise and in a way to remind for which repositories they come as a dependency. This mirrors the capabilities of existing tools such as **`just-import-git`**(1) and extends them by features such as implicitly allowing multiple imports to take place and supporting imports from sources other than Git repositories (e.g., distfiles, local clones, repositories under arbitrary version control systems). By default, the final configuration has the repositories deduplicated, by merging indistinguishable repositories, other than the `"main"` repository and explicitly stated ones, to a single entry. This mirrors the capability available standalone in **`just-deduplicate-repos`**(1). The tool performs these operations based on a provided **`just-lock-config`**(5) input file and outputs the resulting configuration file at either an explicitly given location or at a location expected by **`just-mr`**(5). OPTIONS ======= **`-h`**, **`--help`** Output a usage message and exit. **`-C`** *`CONFIGFILE`* Use the specified file as the input **`just-lock-config`**(5) file. If not specified, a file with filename `repos.in.json` is searched for in the same _directories_ as **`just-mr`** does when invoked with **`--norc`** when searching for its configuration file. **`-o`** *`CONFIGFILE`* Use the specified file as the output **`just-mr-repository-config`**(5) file. If not specified, a file is searched for in the same way **`just-mr`** does when invoked with **`--norc`**. If none found, it is a file with filename `repos.json` in the parent directory of the input configuration file. **`--local-build-root`** *`PATH`* Root for local CAS, cache, and build directories. The path will be created if it does not exist already. Default: path *`".cache/just"`* in user's home directory. **`--just`** *`PATH`* Path to the **`just`** binary in *`PATH`* or path to the **`just`** binary. Default: *`"just"`*. **`-L`**, **`--local-launcher`** *`JSON_ARRAY`* JSON array with the list of strings representing the launcher to prepend any commands being executed locally. Default: *`["env", "--"]`*. **`--git`** *`PATH`* Path to the git binary in *`PATH`* or path to the git binary. Used when shelling out to git is needed. Default: *`"git"`*. **`--clone`** *`JSON_MAP`* JSON map from filesystem path to pair of repository name and list of bindings. For each map entry, the target repository, found by following the bindings from a given start repository, after all repositories have been imported, will have its workspace root cloned in the specified filesystem directory and its description in the output configuration made to point to this clone. The start repository names must be known, i.e., an initial repository or declared import, and both the start and target repositories will be kept during deduplication. See also ======== **`git`**(1), **`just-lock-config`**(5), **`just-import-git`**(1), **`just-deduplicate-repos`**(1), **`just-mr-repository-config`**(5), **`just-mr`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-mr-repository-config.5.md000066400000000000000000000262651516554100600272550ustar00rootroot00000000000000% JUST-MR REPOSITORY CONFIG(5) | File Formats Manual NAME ==== just-mr-repository-config - The format of the repository config used by **`just-mr`**(1) DESCRIPTION =========== In order for the **`just-mr`**(1) tool to generate a repository configuration file usable by the **`just`**(1) multi-repository build system, it requires a configuration file describing repositories and their dependencies. The file is read as JSON. Any other serialization describing the same JSON object is equivalent. It is assumed that in JSON objects keys occur at most once; it is implementation defined how repetitions of the same key are treated. Workspace root description -------------------------- A repository's *workspace root description* provides information about the location of source files. It can be an explicit description, given as a JSON object, or an implicit one, given as the global name of another repository, from which the workspace root can be inferred. Explicit workspace roots can be of several types, distinguishable by the value of the key *`"type"`* in the JSON object. Depending on this value, other fields in the object are also supported. ### *`"file"`* It defines as workspace root a directory on the file system. The following fields are supported: - *`"path"`* provides the root directory containing the source files. This entry is mandatory. ### *`"archive"`* / *`"zip"`* They define as workspace root a remote archive. The only difference between the two types is the nature of the archive: a tarball (preferably compressed) in the case of *`"archive"`*, or a compressed zip or 7zip file in the case of *`"zip"`*. The following fields are supported: - *`"content"`* provides the Git blob hash of the archive file. This has to be specified in hex encoding. This entry is mandatory. - *`"fetch"`* specifies the URL to the remote archive. This entry is mandatory. - *`"distfile"`* provides an alternative name for the archive file. This entry is optional. If missing, the basename of the fetch URL is used. - *`"mirrors"`* is an optional list of alternative locations to try to fetch from if contacting the main fetch location fails. This entry is optional. - *`"sha256"`*, - *`"sha512"`* provide optional checksum hashes in order to verify the integrity of the remote site archive. These have to be provided in hex encoding. These checks are only performed if the archive file is actually downloaded from the (potentially untrusted) network and not already available locally. - *`"subdir"`* specifies the subdirectory withing the unpacked archive. This entry is optional. If missing, the root directory of the archive is used. ### *`"foreign file"`* Define a root as consisting of single file with given content at a specific name with specified executable bit. The following fields are supported. - *`"content"`*, *`"fetch"`*, *`"distfile"`*, *`"mirrors"`*, *`"sha256"`*, and *`"sha512"`* specify the file content in the same way as they specify the archive content for an *`"archive"`* repository. - *`"name"`* specifies the name the content should have in the defined root. It has to be a plain file name without implicitly specified subdirs. This field is mandatory. - *`"executable"`* is a boolean indicating whether the fetched file should be provided with the executable bit. Defaults to `false`. ### *`"git"`* It defines as workspace root a part of a Git repository. The following fields are supported: - *`"repository"`* provides the URL of the Git repository. This entry is mandatory. Note that only URLs starting with `/`, `./`, or `file://` are considered file URLs. - *`"commit"`* contains the commit hash. This has to be specified in hex encoding. This entry is mandatory. - *`"branch"`* provides the branch name, with the promise that it contains the aforementioned commit. This entry is mandatory. - *`"mirrors"`* is an optional list of alternative locations to try to fetch from if contacting the main repository fails. This entry is optional. - *`"subdir"`* specifies the subdirectory containing the distribution files. This entry is optional. If missing, the root directory of the Git repository is used. - *`"inherit env"`* provides a list of variables. When `just-mr` shells out to `git`, those variables are inherited from the environment `just-mr` is called within, if set there. ### *`"git tree"`* It defines as workspace root as a fixed `git` tree, given by the corresponding tree identifier. If that tree is not known already to `just-mr`, a specified command will be executed in a fresh directory that is expected to produce the given tree somewhere below the working directory. This type of root is the way builds against sources versioned in arbitrary version-control systems can be carried out. The command then would be a call to the version control system requesting an export at a particular version. As the root is already fully determined by the specified `git` tree identifier, the corresponding action need not be fully isolated. In fact, to check out a repository it might be necessary to provide credentials (which do not matter for the checked-out tree, but differ from user to user). To support this, the description of the root can specify environment variables to inherit from the ambient environment. E.g., `SSH_AUTH_SOCK` can be specified here to support `ssh`-based authentication. The following fields are supported: - *`"id"`* provides the Git tree identifier. This entry is mandatory. - *`"cmd"`* provides a list of strings forming a command which promises that, when executed in an empty directory (anywhere in the file system), creates in that directory a directory structure containing the promised Git tree (either top-level or in some subdirectory). This entry is mandatory. - *`"env"`* provides a map of envariables to be set for executing the command. - *`"inherit env"`* provides a list of variables to be inherited from the environment `just-mr` is called within, if set there. ### *`"distdir"`* It defines as workspace root a directory with the distribution archives of the specified repositories. Usually this root is realized as a Git tree in the Git repository in **`just`**'s local build root. The following fields are supported: - *`"repositories"`* provides a list of global names of repositories. This entry is mandatory. ### *`"computed"`* It defines a computed root, typically to be used as root by other repositories. A computed root is given as the artifacts map of an export target of a content-fixed repository. Computed roots themselves are content fixed; the dependency of computed roots on one another must be cycle free. The following fields are supported: - *`"repo"`* specifies the repository the export target belongs to. This entry is mandatory. - *`"target"`* specifies the export target within the given repository in module-name form. This entry is mandatory. - *`"config"`* specifies the configuration at which to evaluate the export target. This entry is optional and defaults to the empty map. ### *`"tree structure"`* It evaluates to the given root but with all files and symlinks replaced by empty blobs. A tree structure root is given by a content-fixed workspace of a repository. Tree structure roots themselves are content fixed; the dependency of tree structure roots on one another must be cycle free. The following fields are supported: - *`"repo"`* specifies the repository whose workspace root must be evaluated. This entry is mandatory. ### Additional keys The key *`"pragma"`* is reserved for type-specific repository directives which alter the workspace root. It is given as a JSON object. The different workspace roots might support different keys for this object; unsupported keys are always ignored. For a *`"file"`* workspace root the pragma key *`"to_git"`* is supported. If its value is *`true`* then it indicates that the workspace root should be returned as a Git tree. If the root directory is already part of a Git repository, its Git tree identifier is used; otherwise, the workspace root will be realized as a Git tree in the Git repository in **`just`**'s local build root. For all workspace roots except *`"distdir"`*, *`"computed"`*, and *`"tree structure"`*, the pragma key *`"special"`* is supported. If its value is *`"ignore"`* then it indicates that the workspace root should ignore all special (i.e., neither file, executable, nor tree) entries. For a *`"file"`* workspace root or for an *`"archive"`* workspace root a value of *`"resolve-completely"`* indicates that the workspace root should resolve all confined relative symbolic links, while a value of *`"resolve-partially"`* indicates that the workspace root should resolve only the confined relative upwards symbolic links; for a *`"file"`* workspace root these two values imply *`"to_git"`* is *`true`*. For all workspace roots the pragma key *`"absent"`* is supported. If its value is *`true`* then it indicates that an absent root should be generated, i.e., one given only by its Git tree without any explicit witnessing repository. Repository description ---------------------- A *repository description* is defined as a JSON object, containing a *workspace root description*, directory roots and names for targets, rules, and expressions files, and bindings to other repositories. Specifically, the following fields are supported: - *`"repository"`* contains a *workspace root description*. This entry is mandatory. - *`"target_root"`*, - *`"rule_root"`*, - *`"expression_root"`* define the root directories for the targets, rules, and expressions, respectively. If provided, they are passed on expanded to the workspace root of the repository named by their value. - *`"target_file_name"`*, - *`"rule_file_name"`*, - *`"expression_file_name"`* refer to the name of the files containing the targets, rules, and expressions, respectively, located relative to the corresponding root directories. These entries are optional. If provided, they are passed on as-is. - *`"bindings"`* provides a JSON object defining dependencies on other repositories. The object's keys are strings defining local repository names, while the values are the corresponding global names of those repositories. If provided, this entry is passed on as-is. Repository configuration format ------------------------------- The repository configuration format is structured as a JSON object. The following fields are supported: - *`"main"`* contains a JSON string that determines which of the provided repositories is considered the main repository. This entry is optional, and if omitted, it will be omitted in the generated **`just-repository-config`**. - *`"repositories"`* contains a JSON object, where each key is the global name of a repository and its corresponding value is the *repository description*. Additional keys --------------- Any JSON object described in this format might have additional keys besides the ones mentioned. The current strategy of **`just-mr`**(1) is to accept and ignore them. Users should be aware that future versions of this format might give specific meanings to these extra keys. See also ======== **`just`**(1), **`just-mr`**(1), **`just-repository-config`**(5) just-buildsystem-justbuild-b1fb5fa/share/man/just-mr.1.md000066400000000000000000000377551516554100600235770ustar00rootroot00000000000000% JUST-MR(1) | General Commands Manual NAME ==== just-mr - multi-repository configuration tool and launcher for the build tool SYNOPSIS ======== **`just-mr`** \[*`OPTION`*\]... **`mrversion`** **`just-mr`** \[*`OPTION`*\]... {**`setup`**|**`setup-env`**} \[**`--all`**\] \[*`main-repo`*\] **`just-mr`** \[*`OPTION`*\]... **`fetch`** \[**`--all`**\] \[**`--backup-to-remote`**] \[**`-o`** *`fetch-dir`*\] \[*`main-repo`*\] **`just-mr`** \[*`OPTION`*\]... **`update`** \[*`repo`*\]... **`just-mr`** \[*`OPTION`*\]... **`gc-repo`** \[**`--drop-only`**\] **`just-mr`** \[*`OPTION`*\]... **`do`** \[*`JUST_ARG`*\]... **`just-mr`** \[*`OPTION`*\]... {**`version`**|**`describe`**|**`analyse`**|**`build`**|**`install`**|**`install-cas`**|**`add-to-cas`**|**`rebuild`**|**`gc`**} \[*`JUST_ARG`*\]... DESCRIPTION =========== Just-MR is a configuration tool for the multi-repository Just build system. It can be used both standalone and as a launcher for **`just`**(1). The tool performs specific operations, based on the invoked subcommand, on repositories described in a configuration file. All subcommands operate at the level of *workspace roots* deduced from the given repository descriptions. See **`just-mr-repository-config`**(5) for more details on the input format. OPTIONS ======= General options --------------- **`-h`**, **`--help`** Output a usage message and exit. **`-C`**, **`--repository-config`** *`PATH`* Path to the multi-repository configuration file. See **`just-mr-repository-config`**(5) for more details. If no configuration file is specified, **`just-mr`** will look for one in the following order: - *`$WORKSPACE_ROOT/repos.json`* (workspace of the **`just-mr`** invocation) - *`$WORKSPACE_ROOT/etc/repos.json`* (workspace of the **`just-mr`** invocation) - *`$HOME/.just-repos.json`* - *`/etc/just-repos.json`* The default configuration lookup order can be adjusted in the just-mrrc file. See **`just-mrrc`**(5) for more details. **`--absent`** *`PATH`* Path to a file specifying which repositories are to be considered absent, overriding the values set by the *`"pragma"`* entries in the multi-repository configuration. The file has to contain a JSON array of those repository names to be considered absent. **`-D`**, **`--defines`** *`JSON`* Defines, via an in-line JSON object, an overlay configuration for **`just`**(1); if used as a launcher for a subcommand known to support **`--defines`**, this defines value is forwarded, otherwise it is ignored. If **`-D`** is given several times, the **`-D`** options overlay (in the sense of *`map_union`*) in the order they are given on the command line. **`--local-build-root`** *`PATH`* Root for local CAS, cache, and build directories. The path will be created if it does not exist already. This option overwrites any values set in the **`just-mrrc`**(5) file. Default: path *`".cache/just"`* in user's home directory. **`--checkout-locations`** *`PATH`* Specification file for checkout locations and additional mirrors. This file contains a JSON object with several known keys: - the key *`""`* of key *`"checkouts"`* specifies pairs of repository URLs as keys and absolute paths as values. Currently supported version control is Git, therefore the respective key is *`"git"`*. The paths contained for each repository URL point to existing locations on the filesystem containing the checkout of the respective repository. - the key *`"local mirrors"`*, if given, is a JSON object mapping primary URLs to a list of local (non-public) mirrors. These mirrors are always tried first (in the given order) before any other URL is contacted. - the key *`"preferred hostnames"`*, if given, is a list of strings specifying known hostnames. When fetching from a non-local mirror, URLs with hostnames in the given list are preferred (in the order given) over URLs with other hostnames. - the key *`"extra inherit env"`*, if given, is a list of strings specifying additional variable names to be inherited from the environment (besides the ones specified in *`"inherit env"`* of the respective repository definition). This can be useful, if the local git mirrors use a different protocol (like `ssh` instead of `https`) and hence require different variables to pass the credentials. This options overwrites any values set in the **`just-mrrc`**(5) file. Default: file path *`".just-local.json"`* in user's home directory. **`-L`**, **`--local-launcher`** *`JSON_ARRAY`* JSON array with the list of strings representing the launcher to prepend actions' commands before being executed locally. Default: *`["env", "--"]`*. **`--distdir`** *`PATH`* Directory to look for distfiles before fetching. If given, this will be the first place distfiles are looked for. This option can be given multiple times to specify a list of distribution directories that are used for lookup in the order they appear on the command line. Directories specified via this option will be appended to the ones set in the **`just-mrrc`**(5) file. Default: the single file path *`".distfiles"`* in user's home directory. **`--main`** *`NAME`* The repository to take the target from. **`-f`**, **`--log-file`** *`PATH`* Path to local log file. **`just-mr`** will store the information printed on stderr in the log file along with the thread id and timestamp when the output has been generated. **`--log-limit`** *`NUM`* Log limit (higher is more verbose) in interval \[0,6\] (Default: 3). **`--restrict-stderr-log-limit`** *`NUM`* Restrict logging on console to the minimum of the specified **`--log-limit`** and the value specified in this option. The default is to not additionally restrict the log level at the console. **`--plain-log`** Do not use ANSI escape sequences to highlight messages. **`--log-append`** Append messages to log file instead of overwriting existing. **`--no-fetch-ssl-verify`** Disable the default peer SSL certificate verification step when fetching archives (for which we verify the hash anyway) from remote. **`--fetch-cacert`** *`PATH`* Path to the CA certificate bundle containing one or more certificates to be used to peer verify archive fetches from remote. **`-r`**, **`--remote-execution-address`** *`NAME`*:*`PORT`* Address of a remote execution service. This is used as an intermediary fetch location for archives, between local CAS (or distdirs) and the network. **`--remote-instance-name`** *`NAME`* Value to pass as `instance_name` in the remote execution API. **`-R`**, **`--remote-serve-address`** *`NAME`*:*`PORT`* Address of a **`just`** **`serve`** service. This is used as intermediary fetch location for Git commits, between local CAS and the network. **`--max-attempts`** *`NUM`* If a remote procedure call (rpc) returns `grpc::StatusCode::UNAVAILABLE`, that rpc is retried at most *`NUM`* times. (Default: 1, i.e., no retry). **`--initial-backoff-seconds`** *`NUM`* Before retrying the second time, the client will wait the given amount of seconds plus a jitter, to better distribute the workload. (Default: 1). **`--max-backoff-seconds`** *`NUM`* From the third attempt (included) on, the backoff time is doubled at each attempt, until it exceeds the `max-backoff-seconds` parameter. From that point, the waiting time is computed as `max-backoff-seconds` plus a jitter. (Default: 60) **`--fetch-absent`** Try to make available all repositories, including those marked as absent. This option cannot be set together with **`--compatible`**. **`--compatible`** At increased computational effort, be compatible with the original remote build execution protocol. If a remote execution service address is provided, this option can be used to match the artifacts expected by the remote endpoint. **`--just`** *`PATH`* Name of the just binary in *`PATH`* or path to the just binary. Default: *`"just"`*. **`--rc`** *`PATH`* Path to the just-mrrc file to use. See **`just-mrrc`**(5) for more details. Default: file path *`".just-mrrc"`* in the user's home directory. **`--dump-rc`** *`PATH`* Dump the effective rc, i.e., the rc after overlaying all applicable auxiliary files specified in the `"rc files"` field, to the specified file. In this way, an rc can be made self-contained in preparation for committing it to a repository. **`--git`** *`PATH`* Path to the git binary in *`PATH`* or path to the git binary. Used in the rare instances when shelling out to git is needed. Default: *`"git"`*. **`--norc`** Option to prevent reading any **`just-mrrc`**(5) file. **`-j`**, **`--jobs`** *`NUM`* Number of jobs to run. Default: Number of cores. Authentication options ---------------------- Only TLS and mutual TLS (mTLS) are supported. They mirror the **`just`**(1) options. **`--tls-ca-cert`** *`PATH`* Path to a TLS CA certificate that is trusted to sign the server certificate. **`--tls-client-cert`** *`PATH`* Path to a TLS client certificate to enable mTLS. It must be passed in conjunction with **`--tls-client-key`** and **`--tls-ca-cert`**. **`--tls-client-key`** *`PATH`* Path to a TLS client key to enable mTLS. It must be passed in conjunction with **`--tls-client-cert`** and **`--tls-ca-cert`**. SUBCOMMANDS =========== **`mrversion`** --------------- Print on stdout a JSON object providing version information for this tool itself; the **`version`** subcommand calls the **`version`** subcommand of just. The version information for just-mr is in the same format that also **`just`** uses. **`setup`**|**`setup-env`** --------------------------- These subcommands fetch all required repositories and generate an appropriate multi-repository **`just`** configuration file. The resulting file is stored in CAS and its path is printed to stdout. See **`just-repository-config`**(5) for more details on the resulting configuration file format. If a main repository is provided in the input configuration or on command line, only it and its dependencies are considered in the generation of the resulting multi-repository configuration file. If no main repository is provided, the lexicographical first repository from the configuration is used. To perform the setup for all repositories from the input configuration file, use the **`--all`** flag. The behavior of the two subcommands differs only with respect to the main repository. In the case of **`setup-env`**, the workspace root of the main repository is left out, such that it can be deduced from the working directory when **`just`** is invoked. In this way, working on a checkout of that repository is possible, while having all of its dependencies properly set up. In the case of **`setup`**, the workspace root of the main repository is taken as-is into the output configuration file. fetch ----- This subcommand prepares all archive-type and **`"git tree"`** workspace roots for an offline build by fetching all their required source files from the specified locations given in the input configuration file or ensuring the specified tree is present in the Git cache, respectively. Any subsequent **`just-mr`** or **`just`** invocations containing fetched archive or **`"git tree"`** workspace roots will thus need no further network connections. If a main repository is provided in the input configuration or on command line, only it and its dependencies are considered for fetching. If no main repository is provided, the lexicographical first repository from the configuration is used. To perform the fetch for all repositories from the input configuration file, use the **`--all`** flag. By default the first existing distribution directory is used as the output directory for writing the fetched archives on disk. If no existing distribution directory can be found an error is produced. To define an output directory that is independent of the given distribution directories, use the **`-o`** option. Additionally, and only in *native mode*, the **`--backup-to-remote`** option can be used in combination with the **`--remote-execution-address`** argument to synchronize the locally fetched archives, as well as the **`"git tree"`** workspace roots, with a remote endpoint. update ------ This subcommand updates the specified repositories (possibly none) and prints the resulting updated configuration file to stdout. Currently, **`just-mr`** can only update Git repositories and it will fail if a different repository type is given. The tool also fails if any of the given repository names are not found in the configuration file. For Git repositories, the subcommand will replace the value for the *`"commit"`* field with the commit hash (as a string) found in the remote repository in the specified branch. The output configuration file will otherwise remain the same at the JSON level with the input configuration file. gc-repo ------- This subcommand rotates the generations of the repository cache. Every root used is added to the youngest generation. Therefore upon a call to **`gc-repo`** all roots are cleaned up that were not used since the last **`gc-repo`**. If **`--drop-only`** is given, only the old generations are cleaned up, without rotation. In this way, storage can be reclaimed; this might be necessary as no perfect sharing happens between the repository generations. do -- This subcommand is used as the canonical way of specifying just arguments and calling **`just`** via **`execvp`**(2). Any subsequent argument is unconditionally forwarded to **`just`**. For *known* subcommands (**`version`**, **`describe`**, **`analyse`**, **`build`**, **`install`**, **`install-cas`**, **`add-to-cas`**, **`rebuild`**, **`gc`**), the **`just-mr setup`** step is performed first for those commands accepting a configuration and the produced configuration is prefixed to the provided arguments. The main repository for the **`setup`** step can be provided in the configuration or on the command line. If no main repository is provided, the lexicographical first repository from the configuration is used. All logging arguments given to **`just-mr`** are passed to **`just`** as early arguments. If log files are provided, an unconditional **`--log-append`** argument is passed as well, which ensures no log messages will get overwritten. The **`--local-launcher`** argument is passed to **`just`** as early argument for those *known* subcommands that accept it (build, install, rebuild). The **`--remote-execution-address`**, **`--remote-instance-name`**, **`--compatible`**, and **`--remote-serve-address`** arguments are passed to **`just`** as early arguments for those *known* subcommands that accept them (analyse, build, install-cas, add-to-cas, install, rebuild, traverse). The *authentication options* given to **`just-mr`** are passed to **`just`** as early arguments for those *known* subcommands that accept them, according to **`just`**(1). **`version`**|**`describe`**|**`analyse`**|**`build`**|**`install`**|**`install-cas`**|**`add-to-cas`**|**`rebuild`**|**`gc`** ------------------------------------------------------------------------------------------------------------------------------ This subcommand is the explicit way of specifying *known* just subcommands and calling **`just`** via **`execvp`**(2). The same description as for the **`do`** subcommand applies. **`gc-repo`** ------------- Rotate the repository-root generations. In this way, all repository roots not needed since the the last call to **`gc-repo`** are purged and the corresponding disk space reclaimed. EXIT STATUS =========== The exit status of **`just-mr`** is one of the following values: - 0: the command completed successfully - 64: setup succeeded, but exec failed - 65: any unspecified error occurred in just-mr - 66: unknown subcommand (internal implementation error of **`just-mr`**) - 67: error parsing the command-line arguments - 68: error parsing the configuration - 69: error during fetch - 70: error during update - 71: error during setup Any other exit code that does not have bit 64 set is a status value from **`just`**, if **`just-mr`** is used as a launcher. See **`just`**(1) for more details. See also ======== **`just-mrrc`**(5), **`just-mr-repository-config`**(5), **`just-repository-config`**(5), **`just`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-mrrc.5.md000066400000000000000000000263661516554100600241240ustar00rootroot00000000000000% JUST-MRRC(5) | File Formats Manual NAME ==== just-mrrc - The format of the configuration used by **`just-mr`**(1) DESCRIPTION =========== The file is read as JSON. Any other serialization describing the same JSON object is equivalent. We assume, that in JSON objects, each key occurs at most once; it is implementation defined how repetitions of the same key are treated. Location objects ---------------- A *location* is a JSON object with the keys *`"root"`*, *`"path"`*, and *`"base"`*. The value for key *`"root"`* is either *`"workspace"`*, *`"home"`*, or *`"system"`*, which have the following meanings: - *`"workspace"`* refers to the root of the workspace in which the `just-mr` invocation was issued (not the workspace of the requested main repository). This location root is ignored if **`just-mr`** was not invoked from inside a workspace. - *`"home"`* refers to the user's home directory. - *`"system"`* refers to the system root *`/`*. The value for key *`"path"`* is the relative path of the item to locate within the location root. The value for key *`"base"`* is a relative path within the location root. This path is only relevant for locations of config files. If such a config file contains relative paths, those will be resolved relative to the specified base. If omitted, the default value *`"."`* is used. The just-mrrc format -------------------- The just-mrrc is given by a JSON object. - The value for the key *`"config lookup order"`* is a JSON list of location objects, specifying where to look for multi-repository configurations (see **`just-mr-repository-config`**(5) for more detail). The lookup is performed in the same order the location objects appear in the list. - The value for the key *`"absent"`*, if provided, is a JSON list of location objects to search for a file specifying the list of absent repositories. - The value for the key *`"local build root"`* is a single location object, specifying the path to use as the local build root. For more details, see **`just-mr`**(1). - The value for the key *`"checkout locations"`* is a single location object, specifying the path to the file describing checkout locations and additional mirror locations. For more details, see **`just-mr`**(1). - The value for the key *`"distdirs"`* is a JSON list of location objects, specifying where to look for distribution files (usually collected via the subcommand **`fetch`**). The lookup is performed in the same order the location objects appear in the list. For more details, see **`just-mr`**(1). - The value for the key *`"just"`* is a single location object, specifying the path to the **`just`** binary to use for execution, if **`just-mr`** is used as a launcher. - The value for the key *`"git"`* is a single location object, specifying the path to the git binary to use in the instances when **`just-mr`** needs to shell out. - The value for the key *`"local launcher"`*, if given, is list of strings setting the default for local launcher for **`just-mr`**; command-line arguments take precedence over the value in the configuration file. If the key *`"local launcher"`* is absent, the default *`["env", "--"]`* is assumed. - The value for the key *`"log limit"`*, if given, sets the default value for the log limit, that can be overridden by the command-line options. - The value for the key *`"restrict stderr log limit"`*, if given, sets the default value for the restriction of the log limit on console; this value can be overridden by the command-line option. - The value *`"log files"`*, if given, has to be a list of location objects, specifying additional log files, on top of those specified on the command line. - The value for the key *`"remote execution"`* is a JSON object specifying the remote execution options for **`just-mr`**. For subkey *`"address"`* the value is a string specifying the remote execution address in a NAME:PORT format. For subkey *`"instance name`* the value is a string specifying the remote execution instance name to be used. For subkey *`"compatible"`* the value is a flag which specifies whether the remote endpoint uses the original remote execution protocol. Each subkey value can be overwritten by its corresponding command-line argument. - The value for the key *`"remote serve"`* is a JSON object specifying the remote serve options for **`just-mr`**. For subkey *`"address"`* the value is a string specifying the remote serve address in a NAME:PORT format. Each subkey value can be overwritten by its corresponding command-line argument. - The value for the key *`"authentication"`* is a JSON object specifying client-side authentication options for **`just-mr`**. For subkey *`"ca cert"`* the value is a single location object specifying the path to a TLS CA certificate. For subkey *`"client cert"`* the value is a single location object specifying the path to a TLS client certificate. For subkey *`"client key"`* the value is a single location object specifying the path to a TLS client key. Each subkey value can be overwritten by its corresponding command-line argument. - The value for the key *`"remote-execution properties"`*, if provided, has to be a list of strings. Each entry is forwarded as `--remote-execution-property` to the invocation of the build tool, if **`just-mr`** is used as a launcher. - The value for the key *`"max attempts"`*, if provided, has to be a number. If a remote procedure call (rpc) returns `grpc::StatusCode::UNAVAILABLE`, that rpc is retried at most this number of times. - The value for the key *`"initial backoff seconds"`*, if provided, has to be a number. Before retrying an rpc the second time, the client will wait the given amount of seconds plus a jitter, to better distribute the workload. - The value for the key *`"max backoff seconds"`*, if provided, has to be a number. Normally, on subsequent retries, the backoff time is doubled; this number specifies the maximal time between attempts of an rpc, not counting the jitter. - The value for the key *`"just files"`* is a JSON object. The keys correspond to options that some **`just`** subcommands accept and require a file as argument. For each key, the value is a list of location objects. When **`just-mr`** is used as a launcher and the invoked subcommand is known to support this option, this option is set in the **`just`** invocation with the first matching entry, if any. The supported options are *`"config"`* and *`endpoint-configuration`*. - The value for the key *`"just args"`* is a JSON object. Its keys are **`just`** subcommands and its value is a JSON list of strings. For the corresponding subcommand, these strings are prefixed to the **`just`** argument vector (after all other options provided through the rc file), if **`just-mr`** is used as a launcher. - The value for the key *`"rc files"`*, if given, is a list of location objects. For those location objects that refer to an existing file, this file is read as an additional rc file overlaying the given rc file in the specified order; the value of `"rc files"` in the overlaying files is ignored. In this way, an rc file committed to a repository can be allowed to set a sensible configuration, remote-execution and serve end points, etc. This is particularly useful when simultaneously working on several projects with different settings. - The value for the key *`"invocation log"`* is a JSON object specifying how each invocation should be logged. It supports the following subkeys. - *`"directory"`* A simple location object specifying under which directory the invocations should be logged. If not given, no invocation logging will happen. - *`"project id"`* A path fragment specifying the subdirectory of the given directory; if not specified, `"unknown"` will be used. Under this directory, for each invocation, an invocation-log directory will be created. The *`"project id"`* is given as a separate subkey, in order to allow workspace-specific rc files that are merged in to set this value. - *`"invocation message"`* An additional info message to be shown, followed by the base name of the invocation logging directory. - *`"metadata"`* A file name specifying where in the invocation-log directory the metadata file should be stored. If not given, no metadata file will be written. See **`just-profile`**(5) for details of the format. - *`"context variables"`* An optional list of environment variables, which are captured at invocation time and stored as key-value pairs in the metadata file. These variables do not affect the build in any way. Instead, they are supposed to provide useful context information about the invocation like username, merge-request ID or source branch. - *`"--dump-graph"`* A file name specifying the file in the invocation-log directory for an invocation-specific `--dump-graph` option. - *`"--dump-plain-graph"`* A file name specifying the file in the invocation-log directory for an invocation-specific `--dump-plain-graph` option. - *`"--dump-artifacts-to-build"`* A file name specifying the file in the invocation-log directory for the invocation-specific `--dump-artifacts-to-build` option. - *`"--dump-artifacts"`* A file name specifying in the file in the invocation-log directory for the invocation-specific `--dump-artifacts` option. - *`"--profile"`* A file name specifying the file in invocation-log directory for an invocation-specific `--profile` option. EXAMPLE ======= An example just-mrrc file could look like the following: ``` jsonc { "rc files": [ {"root": "workspace", "path": "rc.json"} , {"root": "workspace", "path": "etc/rc.json"} ] , "config lookup order": [ {"root": "workspace", "path": "repos.json"} , {"root": "workspace", "path": "etc/repos.json"} , {"root": "home", "path": ".just-repos.json"} , {"root": "system", "path": "etc/just-repos.json"} ] , "invocation log": { "directory": {"root": "system", "path": "var/opt/just-invocation"} , "metadata": "metadata.json" , "--dump-graph": "graph.json" , "--profile": "profile.json" } , "absent": [ {"root": "workspace", "path": "etc/absent.json"} , {"root": "home", "path": ".just-absent"} ] , "local build root": {"root": "home", "path": ".cache/just"} , "checkout locations": {"root": "home", "path": ".just-local.json"} , "local launcher": ["env", "--"] , "log limit": 5 , "restrict stderr log limit": 4 , "log files": [{"root": "home", "path": ".log/just/latest-invocation"}] , "distdirs": [{"root": "home", "path": ".distfiles"}] , "just": {"root": "system", "path": "usr/bin/just"} , "git": {"root": "system", "path": "usr/bin/git"} , "remote execution": {"address": "10.0.0.1:8980"} , "remote-execution properties": ["image:development-v1.2.3"] , "just args": { "build": ["-J", "64"] , "install": ["-J", "64", "--remember"] , "install-cas": ["--remember"] } , "just files": { "config": [ {"root": "workspace", "path": "etc/config.json"} , {"root": "home", "path": ".just-config"} ] } } ``` See also ======== **`just-mr`**(1), **`just-mr-repository-config`**(5) just-buildsystem-justbuild-b1fb5fa/share/man/just-profile.5.md000066400000000000000000000073641516554100600246160ustar00rootroot00000000000000% JUST-PROFILE(5) | File Formats Manual NAME ==== just-profile - The format of profile files written by **`just-mr`**(1) and **`just`**(1) DESCRIPTION =========== If profiling is enabled through the *`"invocation log"`* key in the **`just-mrrc`**(5) file, **`just-mr`**(1) can be told to write a metadata file, and the launched **`just`**(1) process to write a profile file. Both files contain a single JSON object. Metadata file -------------- The metadata file contains the following information. - For the key *`"cmdline"`* the argument vector for the process it will **`execv`**(3) to. As with all launches by **`just-mr`**(1), the program is the zeroth entry of the argument vector. - For the key *`"configuration"`* the blob identifier of the **`just-repository-config`**(5) that is passed to the launched process, if such a config is passed. - For the key *`"time"`* the time of the invocation in seconds since the epoch. - For the key *`"context"`* a map of the environment variables requested to be included in the metadata file with the variable name as key and the value of the variable as value (see *`"context variables"`* field of the *`"invocation log"`* key in the **`just-mrrc`**(5) file). Profile file ------------ The profile file contains the following information. - For the key *`"subcommand"`* the subcommand that was executed. - For the key *`"subcommand args"`* the positional arguments that were provided for that subcommand. - For the key *`"target"`* the target that was analysed/built/installed, as full qualified name. - For the key *`"configuration"`* the build configuration in which the target was analysed/built/installed. - For the key *`"remote"`* an object describing the remote-execution configuration. The following keys are used. - If a remote-execution endpoint is used, for the key *`"address"`*, a string of the form `address:port` is given. - For the key *`"properties"`* the remote-execution properties are specified (naturally as an object). - For the key *`"dispatch"`* the dispatch list is given as an array of pairs (arrays of length two) of a property map and an endpoint as `address:port` string. - For the key *`"exit code"`* the exit code of the **`just`**(1) process. - For the key *`"analysis errors"`*, if present, a list of error messages describing each an error that occurred during analysis. - For the key *`"actions"`* an object. For each action that was looked at in the build phase there is an entry, with the key being the action identifier; the identifier is the same as in the **`just-graph-file`**(5) that is written as a result of the `--dump-graph` option. The value is an object with the following entries. - For the key *`"cached"`* a boolean indicating if the action was taken from cache. - For the key *`"time"`* for non-cached actions the build time in seconds. - For the key *`"exit code"`* the exit code of the action, if not 0. - For the key *`"artifacts"`* an object with keys the output paths and values the hashes of the corresponding artifacts as hex-encoded strings. - For they keys *`"stdout"`* and *`"stderr"`* the hashes of stdout and stderr, respectively, as hex-encoded strings, provided the respective output is non-empty. - For the key *`"start time"`* the start time of the invocation in seconds since the epoch. - For the key *`"build start time"`* the start time of the build phase (if any) in seconds since the epoch. - For the key *`"build stop time"`* the stop time of the build phase (if any) in seconds since the epoch. - For the key *`"stop time"`* the stop time of the invocation in seconds since the epoch. See also ======== **`just-mr-rc`**(5), **`just-graph-file`**(5), **`just-mr`**(1), **`just`**(1) just-buildsystem-justbuild-b1fb5fa/share/man/just-repository-config.5.md000066400000000000000000000121371516554100600266320ustar00rootroot00000000000000% JUST REPOSITORY CONFIG(5) | File Formats Manual NAME ==== just-repository-config - The format of the repository config used by **`just`**(1) DESCRIPTION =========== **`just`**'s repository configuration is read as JSON. Any other serialization describing the same JSON object is equivalent. We assume, that in JSON objects, each key occurs at most once; it is implementation defined how repetitions of the same key are treated. File root description --------------------- Each repository can have multiple *`file roots`*. Each file root is defined as a non-empty JSON list with its first element being a string, which determines the type and semantic of the subsequent elements: - *`"file"`* refers to a file root that is located in the file system. The list has to be of length 2 and the second argument contains the path to the file root. - *`"git tree"`* refers to a file root that is available as part of a Git repository. The list has to be of length 2 or 3 with the remaining two elements being: 1. The *`git tree hash`*, which is sufficient to describe the content of an entire tree including its sub-trees and blobs. The tree hash has to be specified in hex encoding. 2. The path to a Git repository on the file system with the promise that it contains the aforementioned *`git tree hash`*; if this entry is missing, the root is considered absent and any target requiring this root has to come from a specified serve end point. - *`"computed"`* refers to a file root that is the result of evaluating an export target of a content-fixed repository. The list has to have length 5 or 6, with the remaining arguments being the global repository name, the module name, the target name, and the configuration, in that order, optionally followed by a pragma object. If the pragma object contains the entry `true` for the key `"absent"`, that root is considered absent, otherwise not. - *`"tree structure"`* refers to the directory structure of a file root. The list has to have length 2 or 3. The second argument contains the global name of the repository which workspace root gets taken for computation. The third argument is an optional pragma object. If the pragma object contains the key `"absent"`, the entry must be a boolean. If the entry is `true`, that root is considered absent, otherwise not. Repository description ---------------------- A single *`repository description`* is defined as a JSON object, which contains *`file roots`*, file names, and bindings to other repositories. Specifically the following fields are supported: - *`"workspace_root"`* contains the *`file root`* where source files are located. If this entry is missing for the main repository, **`just`** will perform the normal workspace root resolution starting from the current working directory. - *`"target_root"`* contains the *`file root`* where the target files are located. If this entry is missing, the workspace root is taken. - *`"target_file_name"`* contains the file name of target files to use. If this entry is missing, the default target file name *`TARGETS`* is used. - *`"rule_root"`* contains the *`file root`* where the rule files are located. If this entry is missing, the target root is taken. - *`"rule_file_name"`* contains the file name of rule files to use. If this entry is missing, the default rule file name *`RULES`* is used. - *`"expression_root"`* contains the *`file root`* where the expression files are located. If this entry is missing, the rule root is taken. - *`"expression_file_name"`* contains the file name of expression files to use. If this entry is missing, the default expression file name *`EXPRESSIONS`* is used. - *`"bindings"`* contains a JSON object that defines bindings to other repositories by mapping local repository names to global ones. The object's key is local name, while the value is a string representing the global name. Note that any other unsupported field is accepted but ignored. There are no guarantees that any yet unsupported field may not become meaningful in future versions. Repository configuration format ------------------------------- The repository configuration format is a JSON object with the following keys: - *`"main"`* contains a string, which defines the repository name to consider by default if not explicitly specified on the command line (i.e., via **`--main`**). This entry is optional and if omitted **`just`** will use the lexicographically first repository, as usual. - *`"repositories"`* contains a JSON object that defines all repositories by mapping global repository names to *repository descriptions* documented above. This entry is optional and if omitted an empty JSON object is used. NOTES ===== Although the repository configuration is human-readable and can be written by hand, in many cases it will be generated by an independent tool. **`just-mr`**(1) is one such tool that can be used for configuration generation, but not necessarily the only one. See also ======== **`just`**(1), **`just-mr`**(1), **`just-mr-repository-config`**(5) just-buildsystem-justbuild-b1fb5fa/share/man/just-serve-config.5.md000066400000000000000000000176701516554100600255460ustar00rootroot00000000000000% JUST SERVE CONFIG(5) | File Formats Manual NAME ==== just-serve-config - The format of the configuration used by the **`serve`** subcommand of **`just`**(1) DESCRIPTION =========== The file is read as JSON. Any other serialization describing the same JSON object is equivalent. We assume, that in JSON objects, each key occurs at most once; it is implementation defined how repetitions of the same key are treated. Location objects ---------------- The general syntax and semantics of a location object are described in **`just-mrrc`**(5). Here we use a restricted form where the value for key *`"root"`* can only be either *`"home"`* or *`"system"`*. This is because `just serve` is not aware of the concept of workspaces. The just-serve configuration format ----------------------------------- The configuration file is given by a JSON object. - The value for the key *`"local build root"`* is a single location object, specifying the path to use as the root for local CAS, cache, and build directories. The path will be created if it does not exist already. - The value for the key *`"repositories"`* is a list of location objects, specifying paths to Git repositories for **`just`** **`serve`** to use as additional object lookup locations. The paths are to be used in the order given and only if requested objects are not found in the local build root. - The value for the key *`"logging"`* is a JSON object specifying logging options. For subkey *`"files"`* the value is a list of location objects, specifying one or more local log files to use. For subkey *`"limit"`* the value is an integer setting the default for the log limit. For subkey *`"restrict stderr limit"`* the value is an integer setting a restriction for the log on stderr. For subkey *`"plain"`* the value is a flag. If set, do not use ANSI escape sequences to highlight messages. For subkey *`"append"`* the value is a flag. If set, append messages to log file instead of overwriting existing. - The value for the key *`"authentication"`* is a JSON object specifying client-side authentication options for **`just`** **`serve`** when communicating with the remote execution endpoint. For subkey *`"ca cert"`* the value is a single location object, specifying the path to a TLS CA certificate. For subkey *`"client cert"`* the value is a single location object, specifying the path to a TLS client certificate. For subkey *`"client key"`* the value is a single location object, specifying the path to a TLS client key. - The value for the key *`"remote service"`* is a JSON object specifying the server arguments for running **`just`** **`serve`** as a service. For subkey *`"interface"`* the value specifies the interface of the service. If unset, the loopback device is used. For subkey *`"port"`* the value specifies the port to which the service is to listen. If unset, the service will choose to the first available one. For subkey *`"pid file"`* the value is a single location object, specifying the path to a file to which the pid should be stored in plain text. If the file exists, it will be overwritten. For subkey *`"info file"`* the value specifies a single location object, specifying the path to a file file to which the used port, interface, and pid should be stored in JSON format. If the file exists, it will be overwritten. For subkey *`"server cert"`* the value is a single location object, specifying the path to a TLS server certificate. For subkey *`"server key"`* the value is a single location object, specifying the path to a TLS server key. - The value for the key *`"execution endpoint"`* is a JSON object specifying the arguments of a remote execution endpoint to be used by **`just`** **`serve`**. For subkey *`"address"`* the value is a string specifying the remote execution address in a NAME:PORT format. For subkey *`"instance name`* the value is a string specifying the remote execution instance name to be used. For subkey *`"client address"`* the value is a string specifying the remote execution address used by the client in a NAME:PORT format. If not provided, it defaults to *`"address"`*. For subkey *`"compatible"`* the value is a flag which specifies whether the remote endpoint uses the original remote execution protocol. If the key *`"execution endpoint"`* is given, the following three keys will be evaluated as well: - *`"max-attempts"`*: the value must be a number specifying the maximum number of attempts to perform when a remote procedure call (to the *`"execution endpoint"`* given) fails because the resource is unavailable. - *`"initial-backoff-seconds"`*: the value must be a number; before retrying the second time, the client will wait the given amount of seconds plus a jitter, to better distribute the workload. - *`"max-backoff-seconds"`*: the value must be a number; from the third attempt (included) on, the backoff time is doubled at each attempt, until it exceeds the `"max-backoff-seconds"` value. From that point, the waiting time is computed as `"max-backoff-seconds"` value plus a jitter. - The value for the key *`"jobs"`* specifies the number of jobs to run. If unset, the number of available cores is used. - The value for the key *`"build"`* is a JSON object specifying arguments used by **`just`** **`serve`** to orchestrate remote builds. For subkey *`"build jobs"`* the value specifies the number of jobs to run during a remote build. If unset, the same value as for outer key *`"jobs"`* is used. For subkey *`"action timeout"`* the value in a number specifying the timeout limit in seconds for actions run during a remote build. If unset, the default value 300 is used. For subkey *`"target-cache write strategy"`* the value has to be one of the values *`"disable"`*, *`"sync"`*, or *`"split"`*. The default is *`"sync"`*, giving the instruction to synchronize artifacts and write target-level cache entries. The value *`"split"`* does the same using blob splitting when synchronizing artifacts, provided it is supported by the remote-execution endpoint. The value *`"disable"`* disables adding new entries to the target-level cache, which defeats the purpose of typical set up to share target-level computations between clients. For the subkey *`"local launcher"`*, if given, the value has to be a list. This list is used as local launcher for the build in the case the serve process acts simultaneously as remote-execution endpoint. If unset (or `null`), the value `["env", "--"]` will be taken as default. EXAMPLE ======= An example serve configuration file could look as follows. ```jsonc { "local build root": {"root": "system", "path": "var/just-serve/root"} , "authentication": { "ca cert": {"root": "system", "path": "etc/just-serve/certs/ca.crt"} , "client cert": {"root": "system", "path": "etc/just-serve/certs/client.crt"} , "client key": {"root": "system", "path": "etc/just-serve/certs/client.key"} } , "remote service": { "interface": "192.0.2.1" , "port": 9999 , "pid file": {"root": "system", "path": "var/run/just-serve/server.pid"} , "server cert": {"root": "system", "path": "etc/just-serve/certs/server.crt"} , "server key": {"root": "system", "path": "etc/just-serve/certs/server.key"} } , "execution endpoint": {"address": "198.51.100.1:8989"} , "repositories": [ {"root": "system", "path": "var/just-serve/repos/third-party-distfiles"} , {"root": "system", "path": "var/just-serve/repos/project-foo"} , {"root": "system", "path": "var/just-serve/repos/project-bar"} ] , "logging": {"files": [{"root": "home", "path": ".log/just-serve/latest"}]} , "jobs": 8 , "build": {"build jobs": 128} , "max-attempts": 10 , "initial-backoff-seconds": 10 , "max-backoff-seconds": 60 } ``` See also ======== **`just`**(1), **`just-mrrc`**(5) just-buildsystem-justbuild-b1fb5fa/share/man/just.1.md000066400000000000000000001065311516554100600231500ustar00rootroot00000000000000% JUST(1) | General Commands Manual NAME ==== just - a generic build tool SYNOPSIS ======== **`just`** **`version`** **`just`** {**`analyse`**|**`build`**} \[*`OPTION`*\]... \[\[*`module`*\] *`target`*\] **`just`** **`install`** \[*`OPTION`*\]... **`-o`** *`OUTPUT_DIR`* \[\[*`module`*\] *`target`*\] **`just`** **`install-cas`** \[*`OPTION`*\]... *`OBJECT_ID`* **`just`** **`add-to-cas`** \[*`OPTION`*\]... *`PATH`* **`just`** **`describe`** \[*`OPTION`*\]... \[\[*`module`*\] *`target`*\] **`just`** **`rebuild`** \[*`OPTION`*\]... \[\[*`module`*\] *`target`*\] **`just`** **`traverse`** \[*`OPTION`*\]... **`-o`** *`OUTPUT_DIR`* **`-g`** *`GRAPH_FILE`* **`just`** **`gc`** \[*`OPTION`*\]... **`just`** **`execute`** \[*`OPTION`*\]... **`just`** **`serve`** *`SERVE_CONFIG_FILE`* DESCRIPTION =========== Just is a generic multi-repository build system; language-specific knowledge is described in separate rule files. For every build action, the relative location of the inputs is independent of their physical location. This staging allows taking sources from different locations (logical repositories), including bare Git repositories. Targets are defined using JSON format, in proper files (by default, named *`TARGETS`*). Targets are uniquely identified by their name, the repository, and the module they belong to. A module is the relative path from the repository target root directory to a subdirectory containing a target file. The module's name of the targets defined at the target root level is the empty string. Specifying the correct repository, target root, module, and target name allows to process that target independently of the current working directory. If the module is not specified on the command line, **`just`** sets the module corresponding to the current working directory. If a target is not specified, the lexicographically-first target, according to native byte order, is used. So, a target named with an empty string will always be the default target for that module. If a target depends on other targets defined in other modules or repositories, **`just`** will recursively visit all and only the required modules. The main repository is the repository containing the target specified on the command line. The main repository can either be read from the multi-repository configuration file if it contains the key *`"main"`* or through the option **`--main`**. The command-line option **`--main`** overrides what is eventually read from the multi-repository configuration file. If neither the multi-repository configuration file contains the *`"main"`* key nor the **`--main`** option is provided, the lexicographical first repository from the multi-repository configuration file is used as main. The *`workspace_root`* of the main repository is then defined as follows. If the option **`--workspace-root`** is provided, then *`workspace_root`* is set accordingly. If the option is not provided, **`just`** checks if it is specified within the multi-repository configuration file. If it is, then it is set accordingly. If not, **`just`** starts looking for a marker in the current directory first, then in all the parent directories until it finds one. The supported markers are - *`ROOT`* file (can be empty, content is ignored) - *`WORKSPACE`* (can be empty, content is ignored) - *`.git`* (can be either a file - empty or not, content is ignored - or the famous directory) If it fails, **`just`** errors out. For non-main repositories, the *`workspace_root`* entry must be declared in the multi-repository configuration file. Afterwards, *`target_root`*, *`rule_root`*, and *`expression_root`* directories for the main repository are set using the following strategy. If the corresponding command-line option is specified, then it is honored. Otherwise, it is read from the multi-repo configuration file. If it is not specified there, the default value is used. The default value of *`target_root`* is *`workspace_root`*, of *`rule_root`* is *`target_root`*, and of *`expression_root`* is *`rule_root`*. Finally, the file names where *targets*, *rules*, and *expressions* are defined for the main repository. If the corresponding key is present in the multi-repository configuration file, it is set accordingly. If the user gives the corresponding command-line option, it overwrites what is eventually read from the configuration file. If they are not explicitly stated neither within the multi-repository configuration file nor on the command line, the default values are used, which are *`TARGETS`*, *`RULES`*, and *`EXPRESSIONS`*, respectively. For non-main repositories (i.e., repositories defining targets required by the target given on the command line), *`target_root`*, *`rule_root`*, *`expression_root`*, and file names of targets, rules and expressions, if different from the default values, must be declared in the multi-repository configuration file. SUBCOMMANDS =========== **`version`** ------------- Print on stdout a JSON object providing version information about the version of the tool used. This JSON object will contain at least the following keys. - *`"version"`* The version, as a list of numbers of length at least 3, following the usual convention that version numbers are compared lexicographically. - *`"suffix"`* The version suffix as a string. Generally, suffixes starting with a + symbol are positive offsets to the version, while suffixes starting with a *`~`* symbol are negative offsets. - *`"SOURCE_DATE_EPOCH"`* Either a number or *`null`*. If it is a number, it is the time, in seconds since the epoch, of the last commit that went into this binary. It is *`null`* if that time is not known (e.g., in development builds). **`analyse`**|**`build`**|**`install`** --------------------------------------- The subcommands **`analyse`**, **`build`**, and **`install`** are strictly related. In fact, from left to right, one is a subset of the other. **`build`** performs work on top of **`analyse`**, and **`install`** on top of **`build`**. When a user issues **`build`**, the **`analyse`** is called underneath. In particular, there is no need to run these three subcommands sequentially. ### **`analyse`** analyse reads the target graph from *`TARGETS`* files for the given target, computes the action graph (required by e.g., **`build`**, **`install`**, **`traverse`**), and reports the artifacts, provides, and runfiles of the analysed target. In short, the **`analyse`** subcommand identifies all the steps required to **`build`** a given target without actually performing those steps. This subcommand, issued with proper flags, can dump in JSON format artifacts, action graph, nodes, actions, (transitive) targets (both named and anonymous), and trees. ### **`build`** This subcommand performs the actions contained in the action graph computed through the **`analyse`** phase. If building locally, the building process is performed in temporary separate directories to allow for staging according to the logical path described in the *`TARGETS`* file. Since artifacts are only stored in the CAS, the user has to use either the **`install`** or **`install-cas`** subcommand to get them. **`just`** allows for both local (i.e., on the same machine where **`just`** is used) and remote compilation (i.e., by sending requests over a TCP connection, e.g., to a different machine, cluster or cloud infrastructure). In case of a remote compilation, artifacts are compiled remotely and stored in the remote CAS. **`install`** and **`install-cas`** subcommands can be used to locally fetch and stage the desired artifacts. ### **`install`** The **`install`** subcommand determines which (if any) actions need to be (re)done and issues the command to (re)run them. Then, it installs the artifacts (stored in the local or remote CAS) of the processed target under the given *`OUTPUT_DIR`* (set by option **`-o`**) honoring the logical path (aka, staging). If the output path does not exist, it will create all the necessary folders and subfolders. If files are already present, they will be overwritten. **`rebuild`** ------------- This subcommand inspects if builds are fully reproducible or not (e.g., time stamps are used). It simply rebuilds and compares artifacts to the cached build reporting actions with different output. To do so in a meaningful way, it requires that previous build is already in the cache (local or remote). **`describe`** -------------- The **`describe`** subcommand allows for describing the rule generating a target. The rule is resolved in precisely the same way as during the analysis. The doc-strings (if any) from the rule definition (if user-defined) are reported, together with a summary of the declared fields and their types. The multi-repository configuration is honored in the same way as during **`analyse`** and **`build`**; in particular, the rule definition can also reside in a git-tree root. **`install-cas`** ----------------- **`install-cas`** fetches artifacts from CAS (Content Addressable Storage) by means of their *`OBJECT_ID`* (object identifier). The canonical format of an object identifier is *`[::]`*; however, when parsing an object identifier, **`install-cas`** uses the following default rules, to make usage simpler. - The square brackets are optional. - If the size is missing (e.g., because the argument contains no colon), or cannot be parsed as a number, this is not an error, and the value 0 is assumed. While this is almost never the correct size, many CAS implementations, including the local CAS of just itself, ignore the size for lookups. - From the type, only the first letter (*`f`* for non-executable file, *`x`* for executable file, and *`t`* for tree) is significant; the rest is ignored. If the type is missing (e.g., because the argument contains less than two colons), or its first letter is not one of the valid ones, *`f`* is assumed. Depending on whether the output path is set or not, the behavior is different. ### Output path is omitted If the output path is omitted, it prints the artifact content to stdout and if the artifact is a tree, it will print a human readable description. ### Output path is set 1. Output path does not exist The artifact will be staged to that path. If artifact is a file, the installed one will have the name of the output path. If the artifact is a tree, it will create a directory named like the output path, and will stage all the entries (subtrees included) under that directory. 2. Output path exists and it is a directory If the artifact is a tree, a directory named with the hash of tree itself is created under the output path, and all the entries and subtrees are installed inside the hash-named directory. If the artifact is a file, it is installed under the output path and named according to the hash of the artifact itself. 3. Output path exists and it is a file If the artifact is a file, it will replace the existing file. If the artifact is a tree, it will cause an error. **`add-to-cas`** ---------------- **`add-to-cas`** adds a file or directory to the local CAS and reports the hash (without size or type information) on stdout. If a remote endpoint is given, the object is also uploaded there. A main use case of this command is to simplify the setup of `"git tree"` repositories, where it can also avoid checking out a repository of a foreign version-control system twice. **`traverse`** -------------- It allows for the building and staging of requested artifacts from a well-defined *`GRAPH_FILE`*. See **`just-graph-file`**(5) for more details. **`gc`** -------- The **`gc`** subcommand triggers garbage collection of the local cache. More precisely, it rotates the cache and CAS generations. During a build, upon cache hit, everything related to that cache hit is uplinked to the youngest generation; therefore, upon a call to **`gc`** everything not referenced since the last call to **`gc`** is purged and the corresponding disk space reclaimed. Additionally, and before doing generation rotation, - left-over temporary directories (e.g., from interrupted `just` invocations) are removed, and - large files are split and only the chunks and the information how to assemble the file from the chunks are kept; in this way disk space is saved without losing information. As the non-rotating tasks can be useful in their own right, the `--no-rotate` option can be used to request only the clean-up tasks that do not lose information. If it is necessary to remove the entire cache, the `--all` option can be used to skip generation rotation and splitting of large files. In this scenario, all cache generations get removed starting from the oldest generation. `--no-rotate` and `--all` are incompatible options. **`execute`** ------------- This subcommand starts a single node remote execution service, honoring the just native remote protocol. If the flag **`--compatible`** is provided, the execution service will honor the original remote build execution protocol. **`serve`** ----------- This subcommand starts a service that provides target dependencies needed for a remote execution build. It expects as its only and mandatory argument the path to a configuration file, following the format described in **`just-serve-config`**(5). OPTIONS ======= Generic program information --------------------------- **`-h`**, **`--help`** Output a usage message and exit. Supported by: all subcommands. Compatibility options --------------------- **`--compatible`** At increased computational effort, be compatible with the original remote build execution protocol. As the change affects identifiers, the flag must be used consistently for all related invocations. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse|execute. Build configuration options --------------------------- **`--action-timeout`** *`NUM`* Action timeout in seconds. (Default: 300). The timeout is honored only for the remote build. Supported by: analyse|build|install|rebuild|traverse. **`-c`**, **`--config`** *`PATH`* Path to configuration file. Supported by: analyse|build|describe|install|rebuild. **`-C`**, **`--repository-config`** *`PATH`* Path to configuration file for multi-repository builds. See **`just-repository-config`**(5) for more details. Supported by: analyse|build|describe|install|rebuild|traverse. **`-D`**, **`--defines`** *`JSON`* Defines, via an in-line JSON object a configuration to overlay (in the sense of *`map_union`*) to the configuration obtained by the **`--config`** option. If **`-D`** is given several times, the **`-D`** options overlay in the order they are given on the command line. Supported by: analyse|build|describe|install|rebuild. **`--request-action-input`** *`ACTION`* Modify the request to be, instead of the analysis result of the requested target, the input stage of the specified action as artifacts, with empty runfiles and a provides map providing the remaining information about the action, in particular as *`"cmd"`* the arguments vector and *`"env"`* the environment. An action can be specified in the following ways - an action identifier prefixed by the *`%`* character - a number prefixed by the *`#`* character (note that it requires quoting on most shells). This specifies the action with that index of the actions associated directly with that target; the indices start from 0 onwards, and negative indices count from the end of the array of actions. - an action identifier or number without prefix, provided the action identifier does not start with either *`%`* or *`#`* and the number does not happen to be a valid action identifier. Supported by: analyse|build|describe|install|rebuild. **`--expression-file-name`** *`TEXT`* Name of the expressions file. Supported by: analyse|build|describe|install|rebuild. **`--expression-root`** *`PATH`* Path of the expression files' root directory. Default: Same as **`--rule-root`**. Supported by: analyse|build|describe|install|rebuild. **`-L`**, **`--local-launcher`** *`JSON_ARRAY`* JSON array with the list of strings representing the launcher to prepend actions' commands before being executed locally. Default value: *`["env", "--"]`* Supported by: analyse|build|install|rebuild|traverse|execute. **`--local-build-root`** *`PATH`* Root for local CAS, cache, and build directories. The path will be created if it does not exist already. Supported by: add-to-cas|build|describe|install-cas|install|rebuild|traverse|gc|execute. **`--main`** *`NAME`* The repository to take the target from. Supported by: analyse|build|describe|install|rebuild|traverse. **`--rule-file-name`** *`TEXT`* Name of the rules file. Supported by: analyse|build|describe|install|rebuild. **`--rule-root`** *`PATH`* Path of the rule files' root directory. Default: Same as **`--target-root`** Supported by: analyse|build|describe|install|rebuild. **`--target-file-name`** *`TEXT`* Name of the targets file. Supported by: analyse|build|describe|install|rebuild. **`--target-root`** *`PATH`* Path of the target files' root directory. Default: Same as **`--workspace-root`** Supported by: analyse|build|describe|install|rebuild. **`-w`**, **`--workspace-root`** *`PATH`* Path of the workspace's root directory. Supported by: analyse|build|describe|install|rebuild|traverse. General output options ---------------------- **`--dump-artifacts-to-build`** *`PATH`* File path for writing the artifacts to build to. Output format is JSON map with staging path as key, and intentional artifact description as value. Supported by: analyse|build|install|rebuild. **`--dump-artifacts`** *`PATH`* Dump artifacts generated by the given target. Using *`-`* as PATH, it is interpreted as stdout. Note that, passing *`.`*/*`-`* will instead create a file named *`-`* in the current directory. Output format is JSON map with staging path as key, and object id description (hash, type, size) as value. Each artifact is guaranteed to be *`KNOWN`* in CAS. Therefore, this option cannot be used with **`analyse`**. Supported by: build|install|rebuild|traverse. **`--profile`** *`PATH`* Write a profile to the specified path. See **`just-profile`**(5) for details on the format. Supported by: analyse|build|install|rebuild|describe. **`--dump-graph`** *`PATH`* File path for writing the action graph description to. See **`just-graph-file`**(5) for more details. Supported by: analyse|build|install|rebuild. **`--dump-plain-graph`** *`PATH`* File path for writing the action graph description to, however without the additional `"origins"` key. See **`just-graph-file`**(5) for more details. Supported by: analyse|build|install|rebuild. **`-f`**, **`--log-file`** *`PATH`* Path to local log file. **`just`** will store the information printed on stderr in the log file along with the thread id and timestamp when the output has been generated. Supported by: add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute. **`--log-limit`** *`NUM`* Log limit (higher is more verbose) in interval \[0,6\] (Default: 3). Supported by: add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute. **`--restrict-stderr-log-limit`** *`NUM`* Restrict logging on console to the minimum of the specified **`--log-limit`** and the value specified in this option. The default is to not additionally restrict the log level at the console. Supported by: add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute. **`--plain-log`** Do not use ANSI escape sequences to highlight messages. Supported by: add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute. **`--log-append`** Append messages to log file instead of overwriting existing. Supported by: add-to-cas|analyse|build|describe|install|install-cas|rebuild|traverse|gc|execute. **`--expression-log-limit`** *`NUM`* In error messages, truncate the entries in the enumeration of the active environment, as well as the expression to be evaluated, to the specified number of characters (default: 320). Supported by: analyse|build|install. **`--serve-errors-log`** *`PATH`* Path to local file in which **`just`** will write, in machine readable form, the references to all errors that occurred on the serve side. More precisely, the value will be a JSON array with one element per failure, where the element is a pair (array of length 2) consisting of the configured target (serialized, as usual, as a pair of qualified target name an configuration) and a string with the hex representation of the blob identifier of the log; the log itself is guaranteed to be available on the remote-execution side. Supported by: analyse|build|install. **`-P`**, **`--print-to-stdout`** *`LOGICAL_PATH`* After building, print the specified artifact to stdout. Supported by: build|install|rebuild|traverse. **`-p`**, **`--print-unique-artifact`** After building, print the unique artifact to stdout, if any. If the option **`-P`** is given or the number of artifacts is not precisely one, this option has no effect. Supported by: build|install|rebuild|traverse. **`-s`**, **`--show-runfiles`** Do not omit runfiles in build report. Supported by: build|install|rebuild|traverse. **`--target-cache-write-strategy`** *`STRATEGY`* Strategy for creating target-level cache entries. Supported values are - *`sync`* Synchronize the artifacts of the export targets and write target-level cache entries. This is the default behaviour. - *`split`* Synchronize the artifacts of the export targets, using blob splitting if the remote-execution endpoint supports it, and write target-level cache entries. As opposed to the default strategy, additional entries (the chunks) are created in the CAS, but subsequent syncs of similar blobs might need less traffic. - *`disable`* Do not write any target-level cache entries. As no artifacts have to be synced, this can be useful for one-off builds of a project or when the connection to the remote-execution endpoint is behind a very slow network. Supported by: build|install|rebuild. Output dir and path ------------------- **`-o`**, **`--output-dir`** *`PATH`* Path of the directory where outputs will be copied. If the output path does not exist, it will create all the necessary folders and subfolders. If the artifacts have been already staged, they will be overwritten. Required by: install|traverse. **`-o`**, **`--output-path`** *`PATH`* Install path for the artifact. Refer to **`install-cas`** section for more details. Supported by: install-cas. **`--archive`** Instead of installing the requested tree, install an archive with the content of the tree. It is a user error to specify **`--archive`** and not request a tree. Supported by: install-cas. **`--raw-tree`** When installing a tree to stdout, i.e., when no option **`-o`** is given, dump the raw tree rather than a pretty-printed version. This option is ignored if **`--archive`** is given. Supported by: install-cas. **`-P`**, **`--sub-object-path`** *`PATH`* Instead of the specified tree object take the object at the specified logical path inside. Supported by: install-cas. **`--remember`** Ensure that all installed artifacts are available in local CAS as well, even when using remote execution. Supported by: install|traverse|install-cas. Parallelism options ------------------- **`-J`**, **`--build-jobs`** *`NUM`* Number of jobs to run during build phase. Default: same as **`--jobs`**. Supported by: analyse|build|install|rebuild|traverse. **`-j`**, **`--jobs`** *`NUM`* Number of jobs to run. Default: Number of cores. Supported by: analyse|build|describe|install|rebuild|traverse. Remote execution options ------------------------ As remote execution properties shard the target-level cache, they are also available for analysis. In this way, the same action identifiers can be achieved despite the extensional projection inherent to target level caching, e.g., in conjunction with **`--request-action-input`**. **`--remote-execution-property`** *`KEY`*:*`VAL`* Property for remote execution as key-value pair. Specifying this option multiple times will accumulate pairs. If multiple pairs with the same key are given, the latest wins. Supported by: analyse|build|install|rebuild|traverse. **`-r`**, **`--remote-execution-address`** *`NAME`*:*`PORT`* Address of the remote execution service. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse. **`--remote-instance-name`** *`NAME`* Value to pass as `instance_name` in the remote execution API. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse. **`--endpoint-configuration`** FILE File containing a description on how to dispatch to different remote-execution endpoints based on the execution properties. The format is a JSON list of pairs (lists of length two) of an object of strings and a string. The first entry describes a condition (the remote-execution properties have to agree on the domain of this object), the second entry is a remote-execution address in the NAME:PORT format as for the **`-r`** option. The first matching entry (if any) is taken; if none matches, the default execution endpoint is taken (either as specified by **`-r`**, or local execution if no endpoint is specified). Supported by: analyse|build|install|rebuild|traverse. **`--max-attempts`** *`NUM`* If a remote procedure call (rpc) returns `grpc::StatusCode::UNAVAILABLE`, that rpc is retried at most *`NUM`* times. (Default: 1, i.e., no retry). Supported by: analyse|build|describe|install|rebuild|traverse. **`--initial-backoff-seconds`** *`NUM`* Before retrying the second time, the client will wait the given amount of seconds plus a jitter, to better distribute the workload. (Default: 1). Supported by: analyse|build|describe|install|rebuild|traverse. **`--max-backoff-seconds`** *`NUM`* From the third attempt (included) on, the backoff time is doubled at each attempt, until it exceeds the `max-backoff-seconds` parameter. From that point, the waiting time is computed as `max-backoff-seconds` plus a jitter. (Default: 60) Supported by: analyse|build|describe|install|rebuild|traverse. Remote serve options -------------------- **`-R`**, **`--remote-serve-address`** *`NAME`*:*`PORT`* Address of the remote execution service. Supported by: analyse|build|describe|install|rebuild. Authentication options ---------------------- Only TLS and mutual TLS (mTLS) are supported. **`--tls-ca-cert`** *`PATH`* Path to a TLS CA certificate that is trusted to sign the server certificate. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse|execute. **`--tls-client-cert`** *`PATH`* Path to a TLS client certificate to enable mTLS. It must be passed in conjunction with **`--tls-client-key`** and **`--tls-ca-cert`**. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse. **`--tls-client-key`** *`PATH`* Path to a TLS client key to enable mTLS. It must be passed in conjunction with **`--tls-client-cert`** and **`--tls-ca-cert`**. Supported by: add-to-cas|analyse|build|describe|install-cas|install|rebuild|traverse. **`analyse`** specific options ------------------------------ **`--dump-actions`** *`PATH`* Dump actions to file. *`-`* is treated as stdout. Output is a list of action descriptions, in JSON format, for the given target. **`--dump-anonymous`** *`PATH`* Dump anonymous targets to file. *`-`* is treated as stdout. Output is a JSON map, for all transitive targets, with two entries: *`nodes`* and *`rule_maps`*. The former contains maps between node id and the node description. *`rule_maps`* states the maps between the *`mode_type`* and the rule to use in order to make a target out of the node. **`--dump-blobs`** *`PATH`* Dump blobs to file. *`-`* is treated as stdout. The term *`blob`* identifies a collection of strings that the execution back end should be aware of before traversing the action graph. A blob, will be referred to as a *`KNOWN`* artifact in the action graph. **`--dump-nodes`** *`PATH`* Dump nodes of only the given target to file. *`-`* is treated as stdout. Output is a JSON map between node id and its description. **`--dump-vars`** *`PATH`* Dump configuration variables to file. *`-`* is treated as stdout. The output is a JSON list of those variable names (in lexicographic order) at which the configuration influenced the analysis of this target. This might contain variables unset in the configuration if the fact that they were unset (and hence treated as the default *`null`*) was relevant for the analysis of that target. **`--dump-targets`** *`PATH`* Dump all transitive targets to file for the given target. *`-`* is treated as stdout. Output is a JSON map of all targets encoded as tree by their entity name: ``` jsonc { "#": // anonymous targets { "": { "": ["", ...] } // all configs this target is configured with } , "@": // "normal" targets { "": { "": { "": ["", ...] } // all configs this target is configured with } } } ``` **`--dump-export-targets`** *`PATH`* Dump all transitive targets to file for the given target that are export targets. *`-`* is treated as stdout. The output format is the same as for **`--dump-targets`**. **`--dump-targets-graph`** *`PATH`* Dump the graph of configured targets to a file (even if it is called *`-`*). In this graph, only non-source targets are reported. The graph is represented as a JSON object. The keys are the nodes of the graph, and for each node, the value is a JSON object containing the different kind of dependencies (each represented as a list of nodes). - *`"declared"`* are the dependencies coming from the target fields in the definition of the target - *`"implicit"`* are the dependencies implicit from the rule definition - *`"anonymous"`* are the dependencies on anonymous targets implicitly referenced during the evaluation of that rule While the node names are strings (so that they can be keys in a JSON object), they can themselves be decoded as JSON and in this way precisely name the configured target. More precisely, the JSON decoding of a node name is a list of length two, with the first entry being the target name (as *`["@", repo, module, target]`* or _`["#", rule_map_id, node_id]`_) and the second entry the effective configuration. **`--dump-trees`** *`PATH`* Dump trees and all subtrees of the given target to file. *`-`* is treated as stdout. Output is a JSON map between tree ids and the corresponding artifact map, which maps the path to the artifact description. **`--dump-provides`** *`PATH`* Dump the provides map of the given target to file. *`-`* is treated as stdout. The output is a JSON object mapping the providers to their values, serialized as JSON; in particular, artifacts are replaced by a JSON object with their intensional description. Therefore, the dumped JSON is not uniquely readable, but requires an out-of-band understanding where artifacts are to be expected. **`--dump-result`** *`PATH`* Dump the result of the analysis for the requested target to file. *`-`* is treated as stdout. The output is a JSON object with the keys *`"artifacts"`*, *`"provides"`*, and *`"runfiles"`*. **`rebuild`** specific options ------------------------------ **`--vs`** *`NAME`*:*`PORT`*|*`"local"`* Cache endpoint to compare against (use *`"local"`* for local cache). **`--dump-flaky`** *`PATH`* Dump flaky actions to file. **`add-to-cas`** specific options --------------------------------- **`--follow-symlinks`** Resolve the positional argument to not be a symbolic link by following symbolic links. The default is to add the link itself, i.e., the string obtained by **`readlink`**(2), as blob. **`--resolve-special`** *`TEXT`* When adding a directory to CAS, resolve any special filesystem entries based on the strategy denoted by the string value specified. Special entries are all those which are neither file, executables, or directories. If option is missing or a non-supported value is provided, the default behavior is used, in which only non-upwards symlinks are accepted and stored unresolved. Currently accepted values: - *`"ignore"`*: all special entries are ignored - *`"tree-upwards"`*: accept only symlinks with a target path pointing inside the location directory and resolve the ones that are upwards - *`"tree-all"`*: accept only symlinks with a target path pointing inside the location directory and resolve all of them - *`"all"`*: unconditionally accept and resolve all symlinks **`traverse`** specific options ------------------------------- **`-a`**, **`--artifacts`** *`TEXT`* JSON maps between relative path where to copy the artifact and its description (as JSON object as well). **`-g`**, **`--graph-file`** *`TEXT`* *`[[REQUIRED]]`* Path of the file containing the description of the actions. See **`just-graph-file`**(5) for more details. **`--git-cas`** *`TEXT`* Path to a Git repository, containing blobs of potentially missing *`KNOWN`* artifacts. **`describe`** specific options ------------------------------- **`--json`** Omit pretty-printing and describe rule in JSON format. **`--rule`** Module and target arguments refer to a rule instead of a target. **`execute`** specific options ------------------------------ **`-p`**, **`--port`** *`INT`* Execution service will listen to this port. If unset, the service will listen to the first available one. **`--info-file`** *`TEXT`* Write the used port, interface, and pid to this file in JSON format. If the file exists, it will be overwritten. **`-i`**, **`--interface`** *`TEXT`* Interface to use. If unset, the loopback device is used. **`--pid-file`** *`TEXT`* Write pid to this file in plain txt. If the file exists, it will be overwritten. **`--tls-server-cert`** *`TEXT`* Path to the TLS server certificate. **`--tls-server-key`** *`TEXT`* Path to the TLS server key. **`--log-operations-threshold`** *`INT`* Once the number of operations stored exceeds twice *`2^n`*, where *`n`* is given by the option **`--log-operations-threshold`**, at most *`2^n`* operations will be removed, in a FIFO scheme. If unset, defaults to 14. Must be in the range \[0,63\]. **`gc`** specific options ------------------------- **`--no-rotate`** Do not rotate garbage-collection generations. Instead, only carry out clean up tasks that do not affect what is stored in the cache. Incompatible with `--all`. **`--all`** Do not rotate garbage-collection generations and do not split large files. Instead, remove all cache generations at once. Incompatible with `--no-rotate`. EXIT STATUS =========== The exit status of **`just`** is one of the following values: - 0: the command completed successfully - 1: the command failed due to a failing build action - 2: the command successfully parsed all the needed files (e.g., *`TARGETS`*), successfully compiled the eventually required objects, but the generation of some artifacts failed (e.g., a test failed). - 8: the command failed due to an error during analysis (e.g., missing or malformed *`TARGETS`* files, assertion errors during analysis, cyclic dependencies) - 16: the command failed due to some problems related to the build environment (e.g., local build root inaccessible, credentials for remote endpoint not accessible) - 32: the tool was invoked in a syntactically malformed way (e.g., failure parsing the command line) See also ======== **`just-repository-config`**(5), **`just-serve-config`**(5), **`just-graph-file`**(5), **`just-profile`**(5), **`just-mr`**(1) just-buildsystem-justbuild-b1fb5fa/src/000077500000000000000000000000001516554100600204065ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/000077500000000000000000000000001516554100600224035ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/auth/000077500000000000000000000000001516554100600233445ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/auth/TARGETS000066400000000000000000000005121516554100600243760ustar00rootroot00000000000000{ "auth": { "type": ["@", "rules", "CC", "library"] , "name": ["auth"] , "hdrs": ["authentication.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "auth"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/auth/authentication.hpp000066400000000000000000000210121516554100600270700ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_AUTH_AUTHENTICATION_HPP #define INCLUDED_SRC_BUILDTOOL_AUTH_AUTHENTICATION_HPP #include // for errno #include // for strerror() #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" struct Auth final { struct TLS final { class Builder; // CA certificate bundle std::string const ca_cert; // Client-side signed certificate std::string const client_cert; // Client-side private key std::string const client_key; // Server-side signed certificate std::string const server_cert; // Server-side private key std::string const server_key; }; std::variant method; }; class Auth::TLS::Builder final { public: auto SetCACertificate( std::optional cert_file) noexcept -> Builder& { ca_cert_file_ = std::move(cert_file); return *this; } auto SetClientCertificate( std::optional cert_file) noexcept -> Builder& { client_cert_file_ = std::move(cert_file); return *this; } auto SetClientKey(std::optional key_file) noexcept -> Builder& { client_key_file_ = std::move(key_file); return *this; } auto SetServerCertificate( std::optional cert_file) noexcept -> Builder& { server_cert_file_ = std::move(cert_file); return *this; } auto SetServerKey(std::optional key_file) noexcept -> Builder& { server_key_file_ = std::move(key_file); return *this; } /// \brief Finalize building, validate the entries, and create an Auth with /// TLS as method. Validation ensures that either both tls_client_cert or /// tls_client_key are set, or none of the two. /// \return Auth on success, error string on failure, nullopt if no TLS /// configuration fields were set. [[nodiscard]] auto Build() const noexcept -> std::optional> { // To not duplicate default arguments of Auth::TLS in builder, // create a default config and copy default arguments from there. Auth::TLS const default_auth_tls; bool tls_args_exist = false; // Set and validate the CA certification. // If provided, the CA certificate bundle should of course not be empty. auto ca_cert = default_auth_tls.ca_cert; if (ca_cert_file_.has_value()) { if (auto content = read(*ca_cert_file_)) { if (content->empty()) { return unexpected( std::string("Please provide tls-ca-cert")); } ca_cert = *std::move(content); tls_args_exist = true; } else { return unexpected( fmt::format("Could not read '{}' CA certificate.", ca_cert_file_->string())); } } // Set and validate the client-side certification. // To enable mTLS, both tls_client_{certificate,key} must be supplied. auto client_cert = default_auth_tls.client_cert; if (client_cert_file_.has_value()) { if (auto content = read(*client_cert_file_)) { client_cert = *std::move(content); tls_args_exist = true; } else { return unexpected( fmt::format("Could not read '{}' client certificate.", client_cert_file_->string())); } } auto client_key = default_auth_tls.client_key; if (client_key_file_.has_value()) { if (auto content = read(*client_key_file_)) { client_key = *std::move(content); tls_args_exist = true; } else { return unexpected(fmt::format("Could not read '{}' client key.", client_key_file_->string())); } } if (client_cert.empty() != client_key.empty()) { std::string error = client_cert.empty() ? "Please also provide tls-client-cert" : "Please also provide tls-client-key"; return unexpected(std::move(error)); } // Set and validate the server-side certification. // To enable mTLS, both tls_server_{certificate,key} must be supplied. auto server_cert = default_auth_tls.server_cert; if (server_cert_file_.has_value()) { if (auto content = read(*server_cert_file_)) { server_cert = *std::move(content); tls_args_exist = true; } else { return unexpected( fmt::format("Could not read '{}' server certificate.", server_cert_file_->string())); } } auto server_key = default_auth_tls.server_key; if (server_key_file_.has_value()) { if (auto content = read(*server_key_file_)) { server_key = *std::move(content); tls_args_exist = true; } else { return unexpected(fmt::format("Could not read '{}' server key.", server_key_file_->string())); } } if (server_cert.empty() != server_key.empty()) { std::string error = server_cert.empty() ? "Please also provide tls-server-cert" : "Please also provide tls-server-key"; return unexpected(std::move(error)); } // If no TLS arguments were ever set, there is nothing to build. if (not tls_args_exist) { return std::nullopt; } // Return an authentication configuration with mTLS enabled. return Auth{.method = Auth::TLS{.ca_cert = std::move(ca_cert), .client_cert = std::move(client_cert), .client_key = std::move(client_key), .server_cert = std::move(server_cert), .server_key = std::move(server_key)}}; } private: std::optional ca_cert_file_; std::optional client_cert_file_; std::optional client_key_file_; std::optional server_cert_file_; std::optional server_key_file_; /// \brief Auxiliary function to read the content of certification files. [[nodiscard]] static auto read(std::filesystem::path const& x) noexcept -> std::optional { try { // if the file does not exist, it will throw an exception auto file = std::filesystem::canonical(x); std::ifstream cert{file}; if (cert.fail()) { Logger::Log(LogLevel::Error, "Certification file {} failed to open with:\n{}", file.string(), strerror(errno)); return std::nullopt; } std::string tmp((std::istreambuf_iterator(cert)), std::istreambuf_iterator()); return tmp; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); } return std::nullopt; } }; #endif // INCLUDED_SRC_BUILDTOOL_AUTH_AUTHENTICATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/000077500000000000000000000000001516554100600250275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target/000077500000000000000000000000001516554100600301755ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target/TARGETS000066400000000000000000000020211516554100600312240ustar00rootroot00000000000000{ "target": { "type": ["@", "rules", "CC", "library"] , "name": ["target"] , "hdrs": ["analysed_target.hpp"] , "srcs": ["analysed_target.cpp"] , "deps": [ "graph_information" , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] ] , "stage": ["src", "buildtool", "build_engine", "analysed_target"] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/expression", "expression"] ] } , "graph_information": { "type": ["@", "rules", "CC", "library"] , "name": ["graph_information"] , "hdrs": ["target_graph_information.hpp"] , "srcs": ["target_graph_information.cpp"] , "deps": [ ["@", "json", "", "json"] , ["src/buildtool/build_engine/target_map", "configured_target"] ] , "stage": ["src", "buildtool", "build_engine", "analysed_target"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target/analysed_target.cpp000066400000000000000000000057321516554100600340560ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "gsl/gsl" #include "src/buildtool/build_engine/expression/expression.hpp" namespace { void CollectNonKnownArtifacts( ExpressionPtr const& expr, gsl::not_null*> const& artifacts, gsl::not_null*> const& traversed) { if (not traversed->contains(expr)) { if (expr->IsMap()) { auto result_map = Expression::map_t::underlying_map_t{}; for (auto const& [key, val] : expr->Map()) { CollectNonKnownArtifacts(val, artifacts, traversed); } } else if (expr->IsList()) { auto result_list = Expression::list_t{}; result_list.reserve(expr->List().size()); for (auto const& val : expr->List()) { CollectNonKnownArtifacts(val, artifacts, traversed); } } else if (expr->IsNode()) { auto const& node = expr->Node(); if (node.IsAbstract()) { CollectNonKnownArtifacts( node.GetAbstract().target_fields, artifacts, traversed); } else { // value node CollectNonKnownArtifacts(node.GetValue(), artifacts, traversed); } } else if (expr->IsResult()) { auto const& result = expr->Result(); CollectNonKnownArtifacts( result.artifact_stage, artifacts, traversed); CollectNonKnownArtifacts(result.runfiles, artifacts, traversed); CollectNonKnownArtifacts(result.provides, artifacts, traversed); } else if (expr->IsArtifact()) { auto const& artifact = expr->Artifact(); if (not artifact.IsKnown()) { artifacts->emplace_back(artifact); } } traversed->emplace(expr); } } } // namespace auto AnalysedTarget::ContainedNonKnownArtifacts() const -> std::vector { auto artifacts = std::vector{}; auto traversed = std::unordered_set{}; CollectNonKnownArtifacts(Artifacts(), &artifacts, &traversed); CollectNonKnownArtifacts(RunFiles(), &artifacts, &traversed); CollectNonKnownArtifacts(Provides(), &artifacts, &traversed); return artifacts; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target/analysed_target.hpp000066400000000000000000000136641516554100600340660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP #define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP #include #include #include #include #include #include // std::move #include #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" class AnalysedTarget { public: explicit AnalysedTarget(TargetResult result, std::vector actions, std::vector blobs, std::vector trees, std::vector tree_overlays, std::unordered_set vars, std::set tainted, std::set implied_export_targets, TargetGraphInformation graph_information) : result_{std::move(result)}, actions_{std::move(actions)}, blobs_{std::move(blobs)}, trees_{std::move(trees)}, tree_overlays_{std::move(tree_overlays)}, vars_{std::move(vars)}, tainted_{std::move(tainted)}, implied_export_targets_{std::move(implied_export_targets)}, graph_information_{std::move(graph_information)} {} [[nodiscard]] auto Actions() const& noexcept -> std::vector const& { return actions_; } [[nodiscard]] auto Actions() && noexcept -> std::vector { return std::move(actions_); } [[nodiscard]] auto Artifacts() const& noexcept -> ExpressionPtr const& { return result_.artifact_stage; } [[nodiscard]] auto Artifacts() && noexcept -> ExpressionPtr { return std::move(result_.artifact_stage); } [[nodiscard]] auto RunFiles() const& noexcept -> ExpressionPtr const& { return result_.runfiles; } [[nodiscard]] auto RunFiles() && noexcept -> ExpressionPtr { return std::move(result_.runfiles); } [[nodiscard]] auto Provides() const& noexcept -> ExpressionPtr const& { return result_.provides; } [[nodiscard]] auto Provides() && noexcept -> ExpressionPtr { return std::move(result_.provides); } [[nodiscard]] auto Blobs() const& noexcept -> std::vector const& { return blobs_; } [[nodiscard]] auto Trees() && noexcept -> std::vector { return std::move(trees_); } [[nodiscard]] auto Trees() const& noexcept -> std::vector const& { return trees_; } [[nodiscard]] auto TreeOverlays() && noexcept -> std::vector { return std::move(tree_overlays_); } [[nodiscard]] auto TreeOverlays() const& noexcept -> std::vector const& { return tree_overlays_; } [[nodiscard]] auto Blobs() && noexcept -> std::vector { return std::move(blobs_); } [[nodiscard]] auto Vars() const& noexcept -> std::unordered_set const& { return vars_; } [[nodiscard]] auto Vars() && noexcept -> std::unordered_set { return std::move(vars_); } [[nodiscard]] auto Tainted() const& noexcept -> std::set const& { return tainted_; } [[nodiscard]] auto Tainted() && noexcept -> std::set { return std::move(tainted_); } [[nodiscard]] auto ImpliedExport() const& noexcept -> std::set const& { return implied_export_targets_; } [[nodiscard]] auto ImpliedExport() && noexcept -> std::set { return std::move(implied_export_targets_); } [[nodiscard]] auto Result() const& noexcept -> TargetResult const& { return result_; } [[nodiscard]] auto Result() && noexcept -> TargetResult { return std::move(result_); } [[nodiscard]] auto GraphInformation() const& noexcept -> TargetGraphInformation const& { return graph_information_; } [[nodiscard]] auto GraphInformation() && noexcept -> TargetGraphInformation { return std::move(graph_information_); } // Obtain a set of all non-known artifacts from artifacts/runfiles/provides. [[nodiscard]] auto ContainedNonKnownArtifacts() const -> std::vector; private: TargetResult result_; std::vector actions_; std::vector blobs_; std::vector trees_; std::vector tree_overlays_; std::unordered_set vars_; std::set tainted_; std::set implied_export_targets_; TargetGraphInformation graph_information_; }; using AnalysedTargetPtr = std::shared_ptr; #endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP target_graph_information.cpp000066400000000000000000000030351516554100600356770ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include auto TargetGraphInformation::NodeString() const noexcept -> std::optional { if (node_) { return node_->ToString(); } return std::nullopt; } namespace { auto NodesToString(std::vector const& nodes) -> std::vector { std::vector result{}; result.reserve(nodes.size()); for (auto const& n : nodes) { if (n) { result.emplace_back(n->ToString()); } } return result; } } // namespace auto TargetGraphInformation::DepsToJson() const -> nlohmann::json { auto result = nlohmann::json::object(); result["declared"] = NodesToString(direct_); result["implicit"] = NodesToString(implicit_); result["anonymous"] = NodesToString(anonymous_); return result; } target_graph_information.hpp000066400000000000000000000043651516554100600357130ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/analysed_target// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_TARGET_GRAPH_INFORMATION_HPP #define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_TARGET_GRAPH_INFORMATION_HPP #include #include #include // std::move #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" class TargetGraphInformation { public: explicit TargetGraphInformation( BuildMaps::Target::ConfiguredTargetPtr node, std::vector direct, std::vector implicit, std::vector anonymous) : node_{std::move(node)}, direct_{std::move(direct)}, implicit_{std::move(implicit)}, anonymous_{std::move(anonymous)} {} static const TargetGraphInformation kSource; [[nodiscard]] auto Node() const noexcept -> BuildMaps::Target::ConfiguredTargetPtr { return node_; } [[nodiscard]] auto NodeString() const noexcept -> std::optional; [[nodiscard]] auto DepsToJson() const -> nlohmann::json; private: BuildMaps::Target::ConfiguredTargetPtr node_; std::vector direct_; std::vector implicit_; std::vector anonymous_; }; inline const TargetGraphInformation TargetGraphInformation::kSource = TargetGraphInformation{nullptr, {}, {}, {}}; #endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_TARGET_GRAPH_INFORMATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/000077500000000000000000000000001516554100600267615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/TARGETS000066400000000000000000000147741516554100600300320ustar00rootroot00000000000000{ "module_name": { "type": ["@", "rules", "CC", "library"] , "name": ["module_name"] , "hdrs": ["module_name.hpp"] , "deps": [["src/utils/cpp", "hash_combine"]] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "directory_map": { "type": ["@", "rules", "CC", "library"] , "name": ["directory_map"] , "hdrs": ["directory_map.hpp"] , "srcs": ["directory_map.cpp"] , "deps": [ "module_name" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "config"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] , "private-deps": [["@", "fmt", "", "fmt"]] } , "json_file_map": { "type": ["@", "rules", "CC", "library"] , "name": ["json_file_map"] , "hdrs": ["json_file_map.hpp"] , "deps": [ "module_name" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "config"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "targets_file_map": { "type": ["@", "rules", "CC", "library"] , "name": ["targets_file_map"] , "hdrs": ["targets_file_map.hpp"] , "deps": [ "json_file_map" , "module_name" , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "config"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "entity_name_data": { "type": ["@", "rules", "CC", "library"] , "name": ["entity_name_data"] , "hdrs": ["entity_name_data.hpp"] , "srcs": ["entity_name_data.cpp"] , "deps": [ "module_name" , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "entity_name": { "type": ["@", "rules", "CC", "library"] , "name": ["entity_name"] , "hdrs": ["entity_name.hpp"] , "deps": [ "entity_name_data" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "config"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "source_map": { "type": ["@", "rules", "CC", "library"] , "name": ["source_map"] , "hdrs": ["source_map.hpp"] , "srcs": ["source_map.cpp"] , "deps": [ "directory_map" , "entity_name_data" , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/common", "config"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] , "private-deps": [ "module_name" , ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/analysed_target", "graph_information"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] ] } , "field_reader": { "type": ["@", "rules", "CC", "library"] , "name": ["field_reader"] , "hdrs": ["field_reader.hpp"] , "deps": [ "entity_name" , "entity_name_data" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "config"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "expression_function": { "type": ["@", "rules", "CC", "library"] , "name": ["expression_function"] , "hdrs": ["expression_function.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/expression", "linked_map"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "gsl"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "expression_map": { "type": ["@", "rules", "CC", "library"] , "name": ["expression_map"] , "hdrs": ["expression_map.hpp"] , "srcs": ["expression_map.cpp"] , "deps": [ "entity_name_data" , "expression_function" , "json_file_map" , "module_name" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "config"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] , "private-deps": ["field_reader"] } , "user_rule": { "type": ["@", "rules", "CC", "library"] , "name": ["user_rule"] , "hdrs": ["user_rule.hpp"] , "deps": [ "entity_name_data" , "expression_function" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/utils/cpp", "concepts"] , ["src/utils/cpp", "gsl"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] } , "rule_map": { "type": ["@", "rules", "CC", "library"] , "name": ["rule_map"] , "hdrs": ["rule_map.hpp"] , "srcs": ["rule_map.cpp"] , "deps": [ "entity_name_data" , "expression_map" , "json_file_map" , "module_name" , "user_rule" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "config"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "base_maps"] , "private-deps": [ "entity_name" , "expression_function" , "field_reader" , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/directory_map.cpp000066400000000000000000000055471516554100600323410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/multithreading/async_map_consumer.hpp" auto BuildMaps::Base::CreateDirectoryEntriesMap( gsl::not_null const& repo_config, std::size_t jobs) -> DirectoryEntriesMap { auto directory_reader = [repo_config](auto /* unused*/, auto setter, auto logger, auto /* unused */, auto const& key) { auto const* ws_root = repo_config->WorkspaceRoot(key.repository); if (ws_root == nullptr) { (*logger)( fmt::format("Cannot determine workspace root for repository {}", key.repository), /*fatal=*/true); return; } if (ws_root->IsAbsent()) { std::string missing_root = "[unknown]"; auto absent_tree = ws_root->GetAbsentTreeId(); if (absent_tree) { missing_root = *absent_tree; } (*logger)(fmt::format("Would have to read directory entries of " "absent root {}", missing_root), /*fatal=*/true); return; } auto dir_path = key.module.empty() ? "." : key.module; if (not ws_root->IsDirectory(dir_path)) { // Missing directory is fine (source tree might be incomplete), // contains no entries. (*setter)(FileRoot::DirectoryEntries{ FileRoot::DirectoryEntries::pairs_t{}}); return; } if (auto read_dir = ws_root->ReadDirectory(dir_path)) { (*setter)(*std::move(read_dir)); return; } (*logger)(fmt::format("Failed to read directory {} from repository {}", dir_path, key.repository), /*fatal=*/true); }; return AsyncMapConsumer{directory_reader, jobs}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/directory_map.hpp000066400000000000000000000026041516554100600323350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_DIRECTORY_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_DIRECTORY_MAP_HPP #include #include "gsl/gsl" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using DirectoryEntriesMap = AsyncMapConsumer; auto CreateDirectoryEntriesMap( gsl::not_null const& repo_config, std::size_t jobs = 0) -> DirectoryEntriesMap; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_DIRECTORY_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/entity_name.hpp000066400000000000000000000256501516554100600320160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_HPP #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/repository_config.hpp" namespace BuildMaps::Base { static inline auto IsString(const nlohmann::json& x) noexcept -> bool { return x.is_string(); } static inline auto IsString(const ExpressionPtr& x) noexcept -> bool { return x->IsString(); } // for compatibility with ExpressionPtr static inline auto GetList(const nlohmann::json& x) noexcept -> nlohmann::json const& { return x; } static inline auto GetList(const ExpressionPtr& x) noexcept -> Expression::list_t const& { return x->Value()->get(); } static inline auto IsList(const nlohmann::json& x) noexcept -> bool { return x.is_array(); } static inline auto IsList(const ExpressionPtr& x) noexcept -> bool { return x->IsList(); } template auto Size(const T& x) noexcept -> std::size_t { return x.size(); } static inline auto GetString(const nlohmann::json& x) -> std::string { return x.template get(); } static inline auto GetString(const ExpressionPtr& x) -> const std::string& { return x->String(); } static inline auto ToString(const nlohmann::json& x) -> std::string { return x.dump(); } static inline auto ToString(const ExpressionPtr& x) noexcept -> std::string { return x->ToString(); } static inline auto IsNone(const nlohmann::json& x) noexcept -> bool { return x.is_null(); } static inline auto IsNone(const ExpressionPtr& x) noexcept -> bool { return x->IsNone(); } template // IsList(list) == true and Size(list) == 2 [[nodiscard]] inline auto ParseEntityName_2(T const& list, EntityName const& current) noexcept -> std::optional { try { if (IsString(list[0]) and IsString(list[1])) { return EntityName{current.GetNamedTarget().repository, GetString(list[0]), GetString(list[1])}; } return std::nullopt; } catch (...) { return std::nullopt; } } template // IsList(list) == true [[nodiscard]] inline auto ParseEntityNameFSReference( std::string const& s0, // list[0] T const& list, std::size_t const list_size, EntityName const& current, std::optional> logger = std::nullopt) noexcept -> std::optional { try { auto get_ref_type = [](std::string const& s) -> ReferenceType { if (s == EntityName::kFileLocationMarker) { return ReferenceType::kFile; } if (s == EntityName::kGlobMarker) { return ReferenceType::kGlob; } if (s == EntityName::kSymlinkLocationMarker) { return ReferenceType::kSymlink; } return ReferenceType::kTree; }; auto const ref_type = get_ref_type(s0); if (list_size == 3) { if (IsString(list[2])) { auto const& name = GetString(list[2]); auto const& x = current.GetNamedTarget(); if (IsNone(list[1])) { return EntityName{x.repository, x.module, name, ref_type}; } if (IsString(list[1])) { auto const& middle = GetString(list[1]); if (middle == "." or middle == x.module) { return EntityName{ x.repository, x.module, name, ref_type}; } } if (logger) { (*logger)( fmt::format("Invalid module name {} for file reference", ToString(list[1]))); } } } return std::nullopt; } catch (...) { return std::nullopt; } } template // IsList(list) == true // list[0] == kRelativeLocationMarker [[nodiscard]] inline auto ParseEntityNameRelative( T const& list, std::size_t const list_size, EntityName const& current, std::optional> logger = std::nullopt) noexcept -> std::optional { try { if (list_size == 3 and IsString(list[1]) and IsString(list[2])) { auto const& relmodule = GetString(list[1]); auto const& name = GetString(list[2]); std::filesystem::path m{current.GetNamedTarget().module}; auto const& module = (m / relmodule).lexically_normal().string(); if (module.compare(0, 3, "../") != 0) { return EntityName{ current.GetNamedTarget().repository, module, name}; } if (logger) { (*logger)(fmt::format( "Relative module name {} is outside of workspace", relmodule)); } } return std::nullopt; } catch (...) { return std::nullopt; } } template // IsList(list) == true // list[0] == EntityName::kLocationMarker [[nodiscard]] inline auto ParseEntityNameLocation( T const& list, std::size_t const list_size, EntityName const& current, gsl::not_null const& repo_config, std::optional> logger = std::nullopt) noexcept -> std::optional { try { if (list_size == 4 and IsString(list[1]) and IsString(list[2]) and IsString(list[3])) { auto const& local_repo_name = GetString(list[1]); auto const& module = GetString(list[2]); auto const& target = GetString(list[3]); auto const* repo_name = repo_config->GlobalName( current.GetNamedTarget().repository, local_repo_name); if (repo_name != nullptr) { return EntityName{*repo_name, module, target}; } if (logger) { (*logger)(fmt::format("Cannot resolve repository name {}", local_repo_name)); } } return std::nullopt; } catch (...) { return std::nullopt; } } template // IsList(list) == true and Size(list) >= 3 [[nodiscard]] inline auto ParseEntityName_3( T const& list, std::size_t const list_size, EntityName const& current, gsl::not_null const& repo_config, std::optional> logger = std::nullopt) noexcept -> std::optional { try { // the first entry of the list must be a string if (IsString(list[0])) { auto const& s0 = GetString(list[0]); if (s0 == EntityName::kRelativeLocationMarker) { return ParseEntityNameRelative( list, list_size, current, logger); } if (s0 == EntityName::kLocationMarker) { return ParseEntityNameLocation( list, list_size, current, repo_config, logger); } if (s0 == EntityName::kAnonymousMarker and logger) { (*logger)(fmt::format( "Parsing anonymous target is not " "supported. Identifiers of anonymous targets should be " "obtained as FIELD value of anonymous fields")); } else if (s0 == EntityName::kFileLocationMarker or s0 == EntityName::kTreeLocationMarker or s0 == EntityName::kGlobMarker or s0 == EntityName::kSymlinkLocationMarker) { return ParseEntityNameFSReference( s0, list, list_size, current, logger); } } return std::nullopt; } catch (...) { return std::nullopt; } } template [[nodiscard]] inline auto ParseEntityName( T const& source, EntityName const& current, gsl::not_null const& repo_config, std::optional> logger = std::nullopt) noexcept -> std::optional { try { std::optional res = std::nullopt; if (IsString(source)) { const auto& x = current.GetNamedTarget(); return EntityName{x.repository, x.module, GetString(source)}; } if (IsList(source)) { auto const& list = GetList(source); const auto list_size = Size(list); if (list_size == 2) { res = ParseEntityName_2(list, current); } else if (list_size >= 3) { res = ParseEntityName_3( list, list_size, current, repo_config, logger); } } if (logger and (res == std::nullopt)) { (*logger)(fmt::format("Syntactically invalid entity name: {}.", ToString(source))); } return res; } catch (...) { return std::nullopt; } } [[nodiscard]] inline auto ParseEntityNameFromJson( nlohmann::json const& json, EntityName const& current, gsl::not_null const& repo_config, std::optional> logger = std::nullopt) noexcept -> std::optional { return ParseEntityName(json, current, repo_config, std::move(logger)); } [[nodiscard]] inline auto ParseEntityNameFromExpression( ExpressionPtr const& expr, EntityName const& current, gsl::not_null const& repo_config, std::optional> logger = std::nullopt) noexcept -> std::optional { return ParseEntityName(expr, current, repo_config, std::move(logger)); } } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/entity_name_data.cpp000066400000000000000000000015311516554100600327720ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" [[nodiscard]] auto BuildMaps::Base::NamedTarget::ToString() const -> std::string { return BuildMaps::Base::EntityName(repository, module, name).ToString(); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/entity_name_data.hpp000066400000000000000000000223601516554100600330020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_DATA_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_DATA_HPP #include #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hash_combine.hpp" namespace BuildMaps::Base { struct AnonymousTarget { ExpressionPtr rule_map; ExpressionPtr target_node; [[nodiscard]] auto operator==(AnonymousTarget const& other) const noexcept -> bool { return rule_map == other.rule_map and target_node == other.target_node; } [[nodiscard]] auto operator<(AnonymousTarget const& other) const noexcept -> bool { if (rule_map < other.rule_map) { return true; } if (other.rule_map < rule_map) { return false; } return target_node < other.target_node; } }; enum class ReferenceType : std::int8_t { kTarget, kFile, kTree, kGlob, kSymlink }; struct NamedTarget { std::string repository; std::string module; std::string name; ReferenceType reference_t{ReferenceType::kTarget}; NamedTarget() = default; NamedTarget(std::string repository, std::string const& module, std::string name, ReferenceType reference_type = ReferenceType::kTarget) : repository{std::move(repository)}, module{normal_module_name(module)}, name{std::move(name)}, reference_t{reference_type} {} static auto normal_module_name(const std::string& module) -> std::string { return std::filesystem::path("/" + module + "/") .lexically_normal() .lexically_relative("/") .parent_path() .string(); } [[nodiscard]] auto ToString() const -> std::string; [[nodiscard]] friend auto operator==(NamedTarget const& x, NamedTarget const& y) noexcept -> bool { return x.repository == y.repository and x.module == y.module and x.name == y.name and x.reference_t == y.reference_t; } [[nodiscard]] friend auto operator!=(NamedTarget const& x, NamedTarget const& y) noexcept -> bool { return not(x == y); } [[nodiscard]] auto operator<(NamedTarget const& other) const noexcept -> bool { if (auto const res = repository <=> other.repository; res != 0) { return res < 0; } if (auto const res = module <=> other.module; res != 0) { return res < 0; } if (auto const res = name <=> other.name; res != 0) { return res < 0; } return reference_t < other.reference_t; } }; class EntityName { public: using variant_t = std::variant; static constexpr auto kLocationMarker = "@"; static constexpr auto kFileLocationMarker = "FILE"; static constexpr auto kTreeLocationMarker = "TREE"; static constexpr auto kGlobMarker = "GLOB"; static constexpr auto kSymlinkLocationMarker = "SYMLINK"; static constexpr auto kRelativeLocationMarker = "./"; static constexpr auto kAnonymousMarker = "#"; EntityName() : EntityName{NamedTarget{}} {} explicit EntityName(variant_t x) : entity_name_{std::move(x)} {} EntityName(std::string repository, const std::string& module, std::string name, ReferenceType reference_type = ReferenceType::kTarget) : EntityName{NamedTarget{std::move(repository), module, std::move(name), reference_type}} {} friend auto operator==(EntityName const& a, EntityName const& b) noexcept -> bool { try { return a.entity_name_ == b.entity_name_; } catch (std::exception const& e) { try { Logger::Log( LogLevel::Error, "Unexpected exception: {}", e.what()); std::terminate(); } catch (...) { std::terminate(); } } catch (...) { std::terminate(); } } friend auto operator<(EntityName const& a, EntityName const& b) noexcept -> bool { try { return a.entity_name_ < b.entity_name_; } catch (std::exception const& e) { try { Logger::Log( LogLevel::Error, "Unexpected exception: {}", e.what()); std::terminate(); } catch (...) { std::terminate(); } } catch (...) { std::terminate(); } } [[nodiscard]] auto IsAnonymousTarget() const -> bool { return std::holds_alternative(entity_name_); } [[nodiscard]] auto IsNamedTarget() const -> bool { return std::holds_alternative(entity_name_); } [[nodiscard]] auto GetAnonymousTarget() -> AnonymousTarget& { return std::get(entity_name_); } [[nodiscard]] auto GetNamedTarget() -> NamedTarget& { return std::get(entity_name_); } [[nodiscard]] auto GetAnonymousTarget() const -> AnonymousTarget const& { return std::get(entity_name_); } [[nodiscard]] auto GetNamedTarget() const -> NamedTarget const& { return std::get(entity_name_); } [[nodiscard]] auto ToJson() const -> nlohmann::json { nlohmann::json j; if (IsAnonymousTarget()) { j.push_back(kAnonymousMarker); const auto& x = GetAnonymousTarget(); j.push_back(x.rule_map.ToIdentifier()); j.push_back(x.target_node.ToIdentifier()); } else { j.push_back(kLocationMarker); const auto& x = GetNamedTarget(); j.push_back(x.repository); if (x.reference_t == ReferenceType::kFile) { j.push_back(kFileLocationMarker); } else if (x.reference_t == ReferenceType::kTree) { j.push_back(kTreeLocationMarker); } else if (x.reference_t == ReferenceType::kGlob) { j.push_back(kGlobMarker); } else if (x.reference_t == ReferenceType::kSymlink) { j.push_back(kSymlinkLocationMarker); } j.push_back(x.module); j.push_back(x.name); } return j; } [[nodiscard]] auto ToString() const -> std::string { return ToJson().dump(); } [[nodiscard]] auto ToModule() const -> ModuleName { const auto& x = GetNamedTarget(); return ModuleName{x.repository, x.module}; } [[nodiscard]] auto IsDefinitionName() const -> bool { return GetNamedTarget().reference_t == ReferenceType::kTarget; } private: variant_t entity_name_; }; } // namespace BuildMaps::Base namespace std { template <> struct hash { [[nodiscard]] auto operator()( const BuildMaps::Base::NamedTarget& t) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, t.repository); hash_combine(&seed, t.module); hash_combine(&seed, t.name); hash_combine(&seed, static_cast(t.reference_t)); return seed; } }; template <> struct hash { [[nodiscard]] auto operator()(const BuildMaps::Base::AnonymousTarget& t) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, t.rule_map); hash_combine(&seed, t.target_node); return seed; } }; template <> struct hash { [[nodiscard]] auto operator()( const BuildMaps::Base::EntityName& t) const noexcept -> std::size_t { if (t.IsAnonymousTarget()) { return hash{}( t.GetAnonymousTarget()); } return hash{}(t.GetNamedTarget()); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_DATA_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/expression_function.hpp000066400000000000000000000121221516554100600335740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" namespace BuildMaps::Base { class ExpressionFunction { public: using Ptr = std::shared_ptr; using imports_t = std::unordered_map>; ExpressionFunction(std::vector vars, imports_t imports, ExpressionPtr expr) noexcept : vars_{std::move(vars)}, imports_{std::move(imports)}, expr_{std::move(expr)} {} [[nodiscard]] auto Evaluate( Configuration const& env, FunctionMapPtr const& functions, std::function const& logger = [](std::string const& error) noexcept -> void { Logger::Log(LogLevel::Error, error); }, std::function annotate_object = [](auto const& /*unused*/) { return std::string{}; }, std::function const& note_user_context = []() noexcept -> void {}) const noexcept -> ExpressionPtr { try { // try-catch to silence clang-tidy's bugprone-exception-escape, // only imports_caller can throw but it is not called here. auto imports_caller = [this, &functions, &annotate_object]( SubExprEvaluator&& /*eval*/, ExpressionPtr const& expr, Configuration const& env) { auto name_expr = expr["name"]; auto const& name = name_expr->String(); auto it = imports_.find(name); if (it != imports_.end()) { std::stringstream ss{}; bool user_context = false; auto result = it->second->Evaluate( env, functions, [&ss](auto const& msg) { ss << msg; }, annotate_object, [&user_context]() { user_context = true; } ); if (result) { return result; } if (user_context) { throw Evaluator::EvaluationError(ss.str(), true, true); } throw Evaluator::EvaluationError( fmt::format( "This call to {} failed in the following way:\n{}", name_expr->ToString(), ss.str()), true); } throw Evaluator::EvaluationError( fmt::format("Unknown expression '{}'.", name)); }; auto newenv = env.Prune(vars_); return expr_.Evaluate( newenv, FunctionMap::MakePtr( functions, "CALL_EXPRESSION", imports_caller), logger, annotate_object, note_user_context); } catch (...) { EnsuresAudit(false); // ensure that the try-block never throws return ExpressionPtr{nullptr}; } } inline static Ptr const kEmptyTransition = std::make_shared( std::vector{}, ExpressionFunction::imports_t{}, Expression::FromJson(R"([{"type": "empty_map"}])"_json)); private: std::vector vars_; imports_t imports_; ExpressionPtr expr_; }; using ExpressionFunctionPtr = ExpressionFunction::Ptr; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/expression_map.cpp000066400000000000000000000110011516554100600325120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" namespace BuildMaps::Base { auto CreateExpressionMap( gsl::not_null const& expr_file_map, gsl::not_null const& repo_config, std::size_t jobs) -> ExpressionFunctionMap { auto expr_func_creator = [expr_file_map, repo_config](auto ts, auto setter, auto logger, auto subcaller, auto const& id) { if (not id.IsDefinitionName()) { (*logger)( fmt::format("{} cannot name an expression", id.ToString()), true); return; } expr_file_map->ConsumeAfterKeysReady( ts, {id.ToModule()}, [repo_config, setter = std::move(setter), logger, subcaller = std::move(subcaller), id](auto json_values) { auto const& target = id.GetNamedTarget(); auto func_it = json_values[0]->find(target.name); if (func_it == json_values[0]->end()) { (*logger)(fmt::format("Cannot find expression {}", EntityName(target).ToString()), true); return; } auto reader = FieldReader::Create( func_it.value(), id, "expression", logger); if (not reader) { return; } auto expr = reader->ReadExpression("expression"); if (not expr) { return; } auto vars = reader->ReadStringList("vars"); if (not vars) { return; } auto import_aliases = reader->ReadEntityAliasesObject("imports", repo_config); if (not import_aliases) { return; } auto [names, ids] = std::move(*import_aliases).Obtain(); auto wrapped_logger = std::make_shared( [logger, id](auto msg, auto fatal) { (*logger)( fmt::format("While handling imports of {}:\n{}", id.ToString(), msg), fatal); }); (*subcaller)( std::move(ids), [setter = std::move(setter), vars = std::move(*vars), names = std::move(names), expr = std::move(expr)](auto const& expr_funcs) { auto imports = ExpressionFunction::imports_t{}; imports.reserve(expr_funcs.size()); for (std::size_t i{}; i < expr_funcs.size(); ++i) { imports.emplace(names[i], *expr_funcs[i]); } (*setter)(std::make_shared( vars, imports, expr)); }, wrapped_logger); }, [logger, id](auto msg, auto fatal) { (*logger)( fmt::format("While reading expression file in {}:\n{}", id.GetNamedTarget().module, msg), fatal); }); }; return ExpressionFunctionMap{expr_func_creator, jobs}; } } // namespace BuildMaps::Base just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/expression_map.hpp000066400000000000000000000044651516554100600325370ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_function.hpp" #include "src/buildtool/build_engine/base_maps/json_file_map.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using ExpressionFileMap = AsyncMapConsumer; [[nodiscard]] static inline auto CreateExpressionFileMap( gsl::not_null const& repo_config, std::size_t jobs) -> JsonFileMap { return CreateJsonFileMap<&RepositoryConfig::ExpressionRoot, &RepositoryConfig::ExpressionFileName, /*kMandatory=*/true>(repo_config, jobs); } using ExpressionFunctionMap = AsyncMapConsumer; auto CreateExpressionMap( gsl::not_null const& expr_file_map, gsl::not_null const& repo_config, std::size_t jobs = 0) -> ExpressionFunctionMap; // use explicit cast to std::function to allow template deduction when used static const std::function kEntityNamePrinter = [](EntityName const& x) -> std::string { return x.ToString(); }; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/field_reader.hpp000066400000000000000000000223111516554100600320760ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { [[nodiscard]] static inline auto GetOrDefault(nlohmann::json const& json, std::string const& key, nlohmann::json&& default_value) -> nlohmann::json { auto value = json.find(key); if (value != json.end()) { return value.value(); } return std::move(default_value); } class FieldReader { public: using Ptr = std::shared_ptr; class EntityAliases { public: [[nodiscard]] auto Obtain() && -> std::pair, std::vector> { return std::make_pair(std::move(names_), std::move(ids_)); } auto reserve(std::size_t size) -> void { names_.reserve(size); ids_.reserve(size); } template auto emplace_back(TName&& name, TId&& id) -> void { names_.emplace_back(std::forward(name)); ids_.emplace_back(std::forward(id)); } private: std::vector names_; std::vector ids_; }; [[nodiscard]] static auto Create(nlohmann::json const& json, EntityName const& id, std::string const& entity_type, AsyncMapConsumerLoggerPtr const& logger) -> std::optional { if (not json.is_object()) { (*logger)(fmt::format("{} definition {} is not an object.", entity_type, id.GetNamedTarget().name), true); return std::nullopt; } return FieldReader(json, id, entity_type, logger); } [[nodiscard]] static auto CreatePtr(nlohmann::json const& json, EntityName const& id, std::string const& entity_type, AsyncMapConsumerLoggerPtr const& logger) -> Ptr { if (not json.is_object()) { (*logger)(fmt::format("{} definition {} is not an object.", entity_type, id.GetNamedTarget().name), true); return nullptr; } return std::make_shared(json, id, entity_type, logger); } [[nodiscard]] auto ReadExpression(std::string const& field_name) const -> ExpressionPtr { auto expr_it = json_.find(field_name); if (expr_it == json_.end()) { (*logger_)(fmt::format("Missing mandatory field {} in {} {}.", field_name, entity_type_, id_.GetNamedTarget().name), true); return ExpressionPtr{nullptr}; } auto expr = Expression::FromJson(expr_it.value()); if (not expr) { (*logger_)( fmt::format("Failed to create expression from JSON:\n {}", json_.dump()), true); } return expr; } [[nodiscard]] auto ReadOptionalExpression( std::string const& field_name, ExpressionPtr const& default_value) const -> ExpressionPtr { auto expr_it = json_.find(field_name); if (expr_it == json_.end()) { return default_value; } auto expr = Expression::FromJson(expr_it.value()); if (not expr) { (*logger_)( fmt::format("Failed to create expression from JSON:\n {}", json_.dump()), true); } return expr; } [[nodiscard]] auto ReadStringList(std::string const& field_name) const -> std::optional> { auto const& list = GetOrDefault(json_, field_name, nlohmann::json::array()); if (not list.is_array()) { (*logger_)(fmt::format("Field {} in {} {} is not a list", field_name, entity_type_, id_.GetNamedTarget().name), true); return std::nullopt; } auto vars = std::vector{}; vars.reserve(list.size()); try { std::transform( list.begin(), list.end(), std::back_inserter(vars), [](auto const& j) { return j.template get(); }); } catch (...) { (*logger_)(fmt::format("List entry in {} of {} {} is not a string", field_name, entity_type_, id_.GetNamedTarget().name), true); return std::nullopt; } return vars; } [[nodiscard]] auto ReadEntityAliasesObject( std::string const& field_name, gsl::not_null const& repo_config) const -> std::optional { auto const& map = GetOrDefault(json_, field_name, nlohmann::json::object()); if (not map.is_object()) { (*logger_)(fmt::format("Field {} in {} {} is not an object", field_name, entity_type_, id_.GetNamedTarget().name), true); return std::nullopt; } auto imports = EntityAliases{}; imports.reserve(map.size()); for (auto const& [key, val] : map.items()) { auto expr_id = ParseEntityNameFromJson( val, id_, repo_config, [this, &field_name, entry = val.dump()]( std::string const& parse_err) { (*logger_)(fmt::format("Parsing entry {} in field {} of {} " "{} failed with:\n{}", entry, field_name, entity_type_, id_.GetNamedTarget().name, parse_err), true); }); if (not expr_id) { return std::nullopt; } imports.emplace_back(key, *expr_id); } return imports; } void ExpectFields(std::unordered_set const& expected) { auto unexpected = nlohmann::json::array(); for (auto const& [key, value] : json_.items()) { if (not expected.contains(key)) { unexpected.push_back(key); } } if (not unexpected.empty()) { (*logger_)(fmt::format("{} {} has unexpected parameters {}", entity_type_, id_.ToString(), unexpected.dump()), false); } } FieldReader(nlohmann::json json, EntityName id, std::string entity_type, AsyncMapConsumerLoggerPtr logger) noexcept : json_(std::move(json)), id_{std::move(id)}, entity_type_{std::move(entity_type)}, logger_{std::move(logger)} {} private: nlohmann::json json_; EntityName id_; std::string entity_type_; AsyncMapConsumerLoggerPtr logger_; }; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/json_file_map.hpp000066400000000000000000000116651516554100600323100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP #include #include #include #include #include // std::move #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using JsonFileMap = AsyncMapConsumer; // function pointer type for specifying which root to get from global config using RootGetter = auto (RepositoryConfig::*)(std::string const&) const -> FileRoot const*; // function pointer type for specifying the file name from the global config using FileNameGetter = auto (RepositoryConfig::*)(std::string const&) const -> std::string const*; template auto CreateJsonFileMap( gsl::not_null const& repo_config, std::size_t jobs) -> JsonFileMap { auto json_file_reader = [repo_config](auto /* unused */, auto setter, auto logger, auto /* unused */, auto const& key) { auto const* root = ((*repo_config).*kGetRoot)(key.repository); auto const* json_file_name = ((*repo_config).*kGetName)(key.repository); if (root == nullptr or json_file_name == nullptr) { (*logger)(fmt::format("Cannot determine root or JSON file name for " "repository {}.", key.repository), true); return; } auto module = std::filesystem::path{key.module}.lexically_normal(); if (module.is_absolute() or *module.begin() == "..") { (*logger)(fmt::format("Modules have to live inside their " "repository, but found {}.", key.module), true); return; } auto json_file_path = module / *json_file_name; if (root->IsAbsent()) { std::string missing_root = "[unknown]"; auto absent_tree = root->GetAbsentTreeId(); if (absent_tree) { missing_root = *absent_tree; } (*logger)(fmt::format( "Would have to read JSON file {} of absent root {}.", json_file_path.string(), missing_root), true); return; } if (not root->IsFile(json_file_path)) { if constexpr (kMandatory) { (*logger)(fmt::format("JSON file {} does not exist.", json_file_path.string()), true); } else { (*setter)(nlohmann::json::object()); } return; } auto const file_content = root->ReadContent(json_file_path); if (not file_content) { (*logger)(fmt::format("cannot read JSON file {}.", json_file_path.string()), true); return; } auto json = nlohmann::json(); try { json = nlohmann::json::parse(*file_content); } catch (std::exception const& e) { (*logger)( fmt::format("JSON file {} does not contain valid JSON:\n{}", json_file_path.string(), e.what()), true); return; } if (not json.is_object()) { (*logger)(fmt::format("JSON in {} is not an object.", json_file_path.string()), true); return; } (*setter)(std::move(json)); }; return AsyncMapConsumer{json_file_reader, jobs}; } } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/module_name.hpp000066400000000000000000000033501516554100600317600ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_MODULE_NAME_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_MODULE_NAME_HPP #include #include #include #include #include "src/utils/cpp/hash_combine.hpp" namespace BuildMaps::Base { struct ModuleName { std::string repository; std::string module; ModuleName(std::string repository, std::string module) : repository{std::move(repository)}, module{std::move(module)} {} [[nodiscard]] auto operator==(ModuleName const& other) const noexcept -> bool { return module == other.module and repository == other.repository; } }; } // namespace BuildMaps::Base namespace std { template <> struct hash { [[nodiscard]] auto operator()( const BuildMaps::Base::ModuleName& t) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, t.repository); hash_combine(&seed, t.module); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_MODULE_NAME_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/rule_map.cpp000066400000000000000000000403021516554100600312700ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/expression_function.hpp" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" namespace BuildMaps::Base { namespace { auto const kRuleFields = std::unordered_set{"anonymous", "artifacts_doc", "config_doc", "config_fields", "config_transitions", "config_vars", "doc", "expression", "field_doc", "implicit", "imports", "provides_doc", "runfiles_doc", "string_fields", "tainted", "target_fields"}; [[nodiscard]] auto ReadAnonymousObject( EntityName const& id, nlohmann::json const& json, gsl::not_null const& repo_config, AsyncMapConsumerLoggerPtr const& logger) -> std::optional { auto obj = GetOrDefault(json, "anonymous", nlohmann::json::object()); if (not obj.is_object()) { (*logger)(fmt::format("Field anonymous in rule {} is not an object", id.GetNamedTarget().name), true); return std::nullopt; } UserRule::anonymous_defs_t anon_defs{}; anon_defs.reserve(obj.size()); for (auto const& [name, def] : obj.items()) { if (not def.is_object()) { (*logger)(fmt::format("Entry {} in field anonymous in rule {} is " "not an object", name, id.GetNamedTarget().name), true); return std::nullopt; } auto target = def.find("target"); if (target == def.end()) { (*logger)(fmt::format("Entry target for {} in field anonymous in " "rule {} is missing", name, id.GetNamedTarget().name), true); return std::nullopt; } if (not target->is_string()) { (*logger)(fmt::format("Entry target for {} in field anonymous in " "rule {} is not a string", name, id.GetNamedTarget().name), true); return std::nullopt; } auto provider = def.find("provider"); if (provider == def.end()) { (*logger)(fmt::format("Entry provider for {} in field anonymous in " "rule {} is missing", name, id.GetNamedTarget().name), true); return std::nullopt; } if (not provider->is_string()) { (*logger)(fmt::format("Entry provider for {} in field anonymous in " "rule {} is not a string", name, id.GetNamedTarget().name), true); return std::nullopt; } auto rule_map = def.find("rule_map"); if (rule_map == def.end()) { (*logger)(fmt::format("Entry rule_map for {} in field anonymous in " "rule {} is missing", name, id.GetNamedTarget().name), true); return std::nullopt; } if (not rule_map->is_object()) { (*logger)(fmt::format("Entry rule_map for {} in field anonymous in " "rule {} is not an object", name, id.GetNamedTarget().name), true); return std::nullopt; } Expression::map_t::underlying_map_t rule_mapping{}; for (auto const& [key, val] : rule_map->items()) { auto rule_name = ParseEntityNameFromJson( val, id, repo_config, [&logger, &id, &name = name](auto msg) { (*logger)( fmt::format("Parsing rule name for entry {} in field " "anonymous in rule {} failed with:\n{}", name, id.GetNamedTarget().name, msg), true); }); if (not rule_name) { return std::nullopt; } rule_mapping.emplace(key, ExpressionPtr{std::move(*rule_name)}); } anon_defs.emplace(name, UserRule::AnonymousDefinition{ .target = target->get(), .provider = provider->get(), .rule_map = ExpressionPtr{ Expression::map_t{std::move(rule_mapping)}}}); } return anon_defs; } [[nodiscard]] auto ReadImplicitObject( EntityName const& id, nlohmann::json const& json, gsl::not_null const& repo_config, AsyncMapConsumerLoggerPtr const& logger) -> std::optional { auto map = GetOrDefault(json, "implicit", nlohmann::json::object()); if (not map.is_object()) { (*logger)(fmt::format("Field implicit in rule {} is not an object", id.GetNamedTarget().name), true); return std::nullopt; } auto implicit_targets = UserRule::implicit_t{}; implicit_targets.reserve(map.size()); for (auto const& [key, val] : map.items()) { if (not val.is_array()) { (*logger)(fmt::format("Entry in implicit field of rule {} is not a " "list.", id.GetNamedTarget().name), true); return std::nullopt; } auto targets = typename UserRule::implicit_t::mapped_type{}; targets.reserve(val.size()); for (auto const& item : val) { auto expr_id = ParseEntityNameFromJson( item, id, repo_config, [&logger, &item, &id](std::string const& parse_err) { (*logger)(fmt::format("Parsing entry {} in implicit field " "of rule {} failed with:\n{}", item.dump(), id.GetNamedTarget().name, parse_err), true); }); if (not expr_id) { return std::nullopt; } targets.emplace_back(*expr_id); } implicit_targets.emplace(key, targets); } return implicit_targets; } [[nodiscard]] auto ReadConfigTransitionsObject( EntityName const& id, nlohmann::json const& json, std::vector const& config_vars, ExpressionFunction::imports_t const& imports, AsyncMapConsumerLoggerPtr const& logger) -> std::optional { auto map = GetOrDefault(json, "config_transitions", nlohmann::json::object()); if (not map.is_object()) { (*logger)( fmt::format("Field config_transitions in rule {} is not an object", id.GetNamedTarget().name), true); return std::nullopt; } auto config_transitions = UserRule::config_trans_t{}; config_transitions.reserve(map.size()); for (auto const& [key, val] : map.items()) { auto expr = Expression::FromJson(val); if (not expr) { (*logger)(fmt::format("Failed to create expression for entry {} in " "config_transitions list of rule {}.", key, id.GetNamedTarget().name), true); return std::nullopt; } config_transitions.emplace( key, std::make_shared(config_vars, imports, expr)); } return config_transitions; } } // namespace auto CreateRuleMap(gsl::not_null const& rule_file_map, gsl::not_null const& expr_map, gsl::not_null const& repo_config, std::size_t jobs) -> UserRuleMap { auto user_rule_creator = [rule_file_map, expr_map, repo_config]( auto ts, auto setter, auto logger, auto /*subcaller*/, auto const& id) { if (not id.IsDefinitionName()) { (*logger)(fmt::format("{} cannot name a rule", id.ToString()), true); return; } rule_file_map->ConsumeAfterKeysReady( ts, {id.ToModule()}, [ts, expr_map, repo_config, setter = std::move(setter), logger, id]( auto json_values) { const auto& target = id.GetNamedTarget(); auto rule_it = json_values[0]->find(target.name); if (rule_it == json_values[0]->end()) { (*logger)(fmt::format("Cannot find rule {} in {}", nlohmann::json(target.name).dump(), nlohmann::json(target.module).dump()), true); return; } auto reader = FieldReader::Create(rule_it.value(), id, "rule", logger); if (not reader) { return; } reader->ExpectFields(kRuleFields); auto expr = reader->ReadExpression("expression"); if (not expr) { return; } auto target_fields = reader->ReadStringList("target_fields"); if (not target_fields) { return; } auto string_fields = reader->ReadStringList("string_fields"); if (not string_fields) { return; } auto config_fields = reader->ReadStringList("config_fields"); if (not config_fields) { return; } auto implicit_targets = ReadImplicitObject( id, rule_it.value(), repo_config, logger); if (not implicit_targets) { return; } auto anonymous_defs = ReadAnonymousObject( id, rule_it.value(), repo_config, logger); if (not anonymous_defs) { return; } auto config_vars = reader->ReadStringList("config_vars"); if (not config_vars) { return; } auto tainted = reader->ReadStringList("tainted"); if (not tainted) { return; } auto import_aliases = reader->ReadEntityAliasesObject("imports", repo_config); if (not import_aliases) { return; } auto [names, ids] = std::move(*import_aliases).Obtain(); expr_map->ConsumeAfterKeysReady( ts, std::move(ids), [id, json = rule_it.value(), expr = std::move(expr), target_fields = std::move(*target_fields), string_fields = std::move(*string_fields), config_fields = std::move(*config_fields), implicit_targets = std::move(*implicit_targets), anonymous_defs = std::move(*anonymous_defs), config_vars = std::move(*config_vars), tainted = std::move(*tainted), names = std::move(names), setter = std::move(setter), logger](auto expr_funcs) { auto imports = ExpressionFunction::imports_t{}; imports.reserve(expr_funcs.size()); for (std::size_t i{}; i < expr_funcs.size(); ++i) { imports.emplace(names[i], *expr_funcs[i]); } auto config_transitions = ReadConfigTransitionsObject( id, json, config_vars, imports, logger); if (not config_transitions) { return; } auto rule = UserRule::Create( target_fields, string_fields, config_fields, implicit_targets, anonymous_defs, config_vars, tainted, std::move(*config_transitions), std::make_shared( config_vars, std::move(imports), std::move(expr)), [&logger](auto const& msg) { (*logger)(msg, true); }); if (rule) { (*setter)(std::move(rule)); } }, [logger, id](auto msg, auto fatal) { (*logger)(fmt::format("While reading expression map " "for rule {}:\n{}", id.GetNamedTarget().ToString(), msg), fatal); }); }, [logger, id](auto msg, auto fatal) { (*logger)(fmt::format("While reading rule file for {}:\n{}", id.GetNamedTarget().ToString(), msg), fatal); }); }; return UserRuleMap{user_rule_creator, jobs}; } } // namespace BuildMaps::Base just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/rule_map.hpp000066400000000000000000000041571516554100600313050ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include "src/buildtool/build_engine/base_maps/json_file_map.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/base_maps/user_rule.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using RuleFileMap = AsyncMapConsumer; [[nodiscard]] static inline auto CreateRuleFileMap( gsl::not_null const& repo_config, std::size_t jobs) -> JsonFileMap { return CreateJsonFileMap<&RepositoryConfig::RuleRoot, &RepositoryConfig::RuleFileName, /*kMandatory=*/true>(repo_config, jobs); } using UserRuleMap = AsyncMapConsumer; auto CreateRuleMap(gsl::not_null const& rule_file_map, gsl::not_null const& expr_map, gsl::not_null const& repo_config, std::size_t jobs = 0) -> UserRuleMap; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/source_map.cpp000066400000000000000000000127501516554100600316270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/source_map.hpp" #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { namespace { auto as_target(const BuildMaps::Base::EntityName& key, ExpressionPtr artifact) -> AnalysedTargetPtr { auto stage = ExpressionPtr{ Expression::map_t{key.GetNamedTarget().name, std::move(artifact)}}; return std::make_shared( TargetResult{.artifact_stage = stage, .provides = Expression::kEmptyMap, .runfiles = stage}, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{}, std::set{}, std::set{}, TargetGraphInformation::kSource); } } // namespace auto CreateSourceTargetMap( const gsl::not_null& dirs, gsl::not_null const& repo_config, HashFunction::Type hash_type, std::size_t jobs) -> SourceTargetMap { auto src_target_reader = [dirs, repo_config, hash_type](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { using std::filesystem::path; const auto& target = key.GetNamedTarget(); auto name = path(target.name).lexically_normal(); if (name.is_absolute() or *name.begin() == "..") { (*logger)( fmt::format("Source file reference outside current module: {}", target.name), true); return; } auto dir = (path(target.module) / name).parent_path(); auto const* ws_root = repo_config->WorkspaceRoot(target.repository); auto src_file_reader = [key, name, setter, logger, dir, ws_root, hash_type]( bool exists_in_ws_root) { if (ws_root != nullptr and exists_in_ws_root) { if (auto desc = ws_root->ToArtifactDescription( hash_type, path(key.GetNamedTarget().module) / name, key.GetNamedTarget().repository)) { (*setter)( as_target(key, ExpressionPtr{std::move(*desc)})); return; } } (*logger)( fmt::format( "Cannot determine source file {} in directory {} of " "repository {}", nlohmann::json( path(key.GetNamedTarget().name).filename().string()) .dump(), nlohmann::json(dir.string()).dump(), nlohmann::json(key.GetNamedTarget().repository).dump()), true); }; if (ws_root != nullptr and ws_root->HasFastDirectoryLookup()) { // by-pass directory map and directly attempt to read from ws_root src_file_reader(ws_root->IsBlob(path(target.module) / name)); return; } dirs->ConsumeAfterKeysReady( ts, {ModuleName{target.repository, dir.string()}}, [key, src_file_reader](auto values) { src_file_reader(values[0]->ContainsBlob( path(key.GetNamedTarget().name).filename().string())); }, [logger, dir](auto msg, auto fatal) { (*logger)( fmt::format("While reading contents of directory {}: {}", nlohmann::json(dir.string()).dump(), msg), fatal); } ); }; return AsyncMapConsumer(src_target_reader, jobs); } } // namespace BuildMaps::Base just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/source_map.hpp000066400000000000000000000031111516554100600316230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_SOURCE_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_SOURCE_MAP_HPP #include #include "gsl/gsl" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using SourceTargetMap = AsyncMapConsumer; auto CreateSourceTargetMap( const gsl::not_null& dirs, gsl::not_null const& repo_config, HashFunction::Type hash_type, std::size_t jobs = 0) -> SourceTargetMap; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_SOURCE_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/targets_file_map.hpp000066400000000000000000000031631516554100600330020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TARGETS_FILE_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TARGETS_FILE_MAP_HPP #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/json_file_map.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Base { using TargetsFileMap = AsyncMapConsumer; [[nodiscard]] static inline auto CreateTargetsFileMap( gsl::not_null const& repo_config, std::size_t jobs) -> JsonFileMap { return CreateJsonFileMap<&RepositoryConfig::TargetRoot, &RepositoryConfig::TargetFileName, /*kMandatory=*/true>(repo_config, jobs); } } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TARGETS_FILE_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/base_maps/user_rule.hpp000066400000000000000000000402061516554100600315010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP #include #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_function.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/utils/cpp/concepts.hpp" #include "src/utils/cpp/gsl.hpp" namespace BuildMaps::Base { // Get duplicates from containers. // NOTE: Requires all input containers to be sorted! // kTriangular=true Performs triangular compare, everyone with everyone. // kTriangular=false Performs linear compare, first with each of the rest. template > [[nodiscard]] static inline auto GetDuplicates(TContainer const& first, TRest const&... rest) -> TResult; template [[nodiscard]] static inline auto JoinContainer(TContainer const& c, std::string const& sep) -> std::string; class UserRule { public: using Ptr = std::shared_ptr; using implicit_t = std::unordered_map>; using implicit_exp_t = std::unordered_map; using config_trans_t = std::unordered_map; struct AnonymousDefinition { std::string target; std::string provider; ExpressionPtr rule_map; }; using anonymous_defs_t = std::unordered_map; [[nodiscard]] static auto Create( std::vector target_fields, std::vector string_fields, std::vector config_fields, implicit_t const& implicit_targets, anonymous_defs_t anonymous_defs, std::vector const& config_vars, std::vector const& tainted, config_trans_t config_transitions, ExpressionFunctionPtr const& expr, std::function const& logger) -> Ptr { auto implicit_fields = std::vector{}; implicit_fields.reserve(implicit_targets.size()); std::transform(implicit_targets.begin(), implicit_targets.end(), std::back_inserter(implicit_fields), [](auto const& el) { return el.first; }); std::sort(implicit_fields.begin(), implicit_fields.end()); auto anonymous_fields = std::vector{}; anonymous_fields.reserve(anonymous_defs.size()); std::transform(anonymous_defs.begin(), anonymous_defs.end(), std::back_inserter(anonymous_fields), [](auto const& el) { return el.first; }); std::sort(anonymous_fields.begin(), anonymous_fields.end()); std::sort(target_fields.begin(), target_fields.end()); std::sort(string_fields.begin(), string_fields.end()); std::sort(config_fields.begin(), config_fields.end()); auto dups = GetDuplicates(kReservedKeywords, target_fields, string_fields, config_fields, implicit_fields, anonymous_fields); if (not dups.empty()) { logger( fmt::format("User-defined fields cannot be any of the reserved " "fields [{}]", JoinContainer(kReservedKeywords, ","))); return nullptr; } dups = GetDuplicates(target_fields, string_fields, config_fields, implicit_fields, anonymous_fields); if (not dups.empty()) { logger( fmt::format("A field can have only one type, but the following " "have more: [{}]", JoinContainer(dups, ","))); return nullptr; } auto transition_targets = std::vector{}; transition_targets.reserve(config_transitions.size()); std::transform(config_transitions.begin(), config_transitions.end(), std::back_inserter(transition_targets), [](auto const& el) { return el.first; }); std::sort(transition_targets.begin(), transition_targets.end()); dups = GetDuplicates(transition_targets, target_fields, implicit_fields, anonymous_fields); if (dups != decltype(dups){transition_targets.begin(), transition_targets.end()}) { logger( fmt::format("Config transitions has to be a map from target " "fields to transition expressions, but found [{}]", JoinContainer(transition_targets, ","))); return nullptr; } auto const setter = [&config_transitions](auto const field) { config_transitions.emplace( field, ExpressionFunction::kEmptyTransition); // wont overwrite }; config_transitions.reserve(target_fields.size() + implicit_fields.size() + anonymous_fields.size()); std::for_each(target_fields.begin(), target_fields.end(), setter); std::for_each(implicit_fields.begin(), implicit_fields.end(), setter); std::for_each(anonymous_fields.begin(), anonymous_fields.end(), setter); implicit_exp_t implicit_target_exp; implicit_target_exp.reserve(implicit_targets.size()); for (auto const& [target_name, target_entity_vec] : implicit_targets) { std::vector target_exps; target_exps.reserve(target_entity_vec.size()); for (auto const& target_entity : target_entity_vec) { target_exps.emplace_back(target_entity); } implicit_target_exp.emplace(target_name, target_exps); } return std::make_shared( std::move(target_fields), std::move(string_fields), std::move(config_fields), implicit_targets, std::move(implicit_target_exp), std::move(anonymous_defs), config_vars, std::set{tainted.begin(), tainted.end()}, std::move(config_transitions), expr); } UserRule(std::vector target_fields, std::vector string_fields, std::vector config_fields, implicit_t implicit_targets, implicit_exp_t implicit_target_exp, anonymous_defs_t anonymous_defs, std::vector config_vars, std::set tainted, config_trans_t config_transitions, ExpressionFunctionPtr expr) noexcept : target_fields_{std::move(target_fields)}, string_fields_{std::move(string_fields)}, config_fields_{std::move(config_fields)}, implicit_targets_{std::move(implicit_targets)}, implicit_target_exp_{std::move(implicit_target_exp)}, anonymous_defs_{std::move(anonymous_defs)}, config_vars_{std::move(config_vars)}, tainted_{std::move(tainted)}, config_transitions_{std::move(config_transitions)}, expr_{std::move(expr)} {} [[nodiscard]] auto TargetFields() const& noexcept -> std::vector const& { return target_fields_; } [[nodiscard]] auto TargetFields() && noexcept -> std::vector { return std::move(target_fields_); } [[nodiscard]] auto StringFields() const& noexcept -> std::vector const& { return string_fields_; } [[nodiscard]] auto StringFields() && noexcept -> std::vector { return std::move(string_fields_); } [[nodiscard]] auto ConfigFields() const& noexcept -> std::vector const& { return config_fields_; } [[nodiscard]] auto ConfigFields() && noexcept -> std::vector { return std::move(config_fields_); } [[nodiscard]] auto ImplicitTargets() const& noexcept -> implicit_t const& { return implicit_targets_; } [[nodiscard]] auto ImplicitTargets() && noexcept -> implicit_t { return std::move(implicit_targets_); } [[nodiscard]] auto ImplicitTargetExps() const& noexcept -> implicit_exp_t const& { return implicit_target_exp_; } [[nodiscard]] auto ExpectedFields() const& noexcept -> std::unordered_set const& { return expected_entries_; } [[nodiscard]] auto ConfigVars() const& noexcept -> std::vector const& { return config_vars_; } [[nodiscard]] auto ConfigVars() && noexcept -> std::vector { return std::move(config_vars_); } [[nodiscard]] auto Tainted() const& noexcept -> std::set const& { return tainted_; } [[nodiscard]] auto Tainted() && noexcept -> std::set { return std::move(tainted_); } [[nodiscard]] auto ConfigTransitions() const& noexcept -> config_trans_t const& { return config_transitions_; } [[nodiscard]] auto ConfigTransitions() && noexcept -> config_trans_t { return std::move(config_transitions_); } [[nodiscard]] auto Expression() const& noexcept -> ExpressionFunctionPtr const& { return expr_; } [[nodiscard]] auto Expression() && noexcept -> ExpressionFunctionPtr { return std::move(expr_); } [[nodiscard]] auto AnonymousDefinitions() const& noexcept -> anonymous_defs_t { return anonymous_defs_; } [[nodiscard]] auto AnonymousDefinitions() && noexcept -> anonymous_defs_t { return std::move(anonymous_defs_); } private: // NOTE: Must be sorted static inline std::vector const kReservedKeywords{ "arguments_config", "tainted", "type"}; static auto ComputeExpectedEntries(std::vector tfields, std::vector sfields, std::vector cfields) -> std::unordered_set { std::size_t n = 0; n += tfields.size(); n += sfields.size(); n += cfields.size(); n += kReservedKeywords.size(); std::unordered_set expected_entries{}; expected_entries.reserve(n); expected_entries.insert(tfields.begin(), tfields.end()); expected_entries.insert(sfields.begin(), sfields.end()); expected_entries.insert(cfields.begin(), cfields.end()); expected_entries.insert(kReservedKeywords.begin(), kReservedKeywords.end()); return expected_entries; } std::vector target_fields_; std::vector string_fields_; std::vector config_fields_; implicit_t implicit_targets_; implicit_exp_t implicit_target_exp_; anonymous_defs_t anonymous_defs_; std::vector config_vars_; std::set tainted_; config_trans_t config_transitions_; ExpressionFunctionPtr expr_; std::unordered_set expected_entries_{ ComputeExpectedEntries(target_fields_, string_fields_, config_fields_)}; }; using UserRulePtr = UserRule::Ptr; namespace detail { template [[nodiscard]] static inline auto MaxSize(TContainer const& first, TRest const&... rest) -> std::size_t { if constexpr (sizeof...(rest) > 0) { return std::max(first.size(), MaxSize(rest...)); } return first.size(); } template static auto inline FindDuplicates(gsl::not_null const& dups, TFirst const& first, TSecond const& second, TRest const&... rest) -> void { ExpectsAudit(std::is_sorted(first.begin(), first.end()) and std::is_sorted(second.begin(), second.end())); std::set_intersection(first.begin(), first.end(), second.begin(), second.end(), std::inserter(*dups, dups->begin())); if constexpr (sizeof...(rest) > 0) { // n comparisons with rest: first<->rest[0], ..., first<->rest[n] FindDuplicates(dups, first, rest...); if constexpr (kTriangular) { // do triangular compare of second with rest // NOLINTNEXTLINE(readability-suspicious-call-argument) FindDuplicates(dups, second, rest...); } } } } // namespace detail template [[nodiscard]] static inline auto GetDuplicates(TContainer const& first, TRest const&... rest) -> TResult { auto dups = TResult{}; constexpr auto kNumContainers = 1 + sizeof...(rest); if constexpr (kNumContainers > 1) { std::size_t size{}; if constexpr (kTriangular) { // worst case if all containers are of the same size size = kNumContainers * detail::MaxSize(first, rest...) / 2; } else { size = std::min(first.size(), detail::MaxSize(rest...)); } dups.reserve(size); detail::FindDuplicates(&dups, first, rest...); } return dups; } template [[nodiscard]] static inline auto JoinContainer(TContainer const& c, std::string const& sep) -> std::string { std::ostringstream oss{}; std::size_t insert_sep{}; for (auto const& i : c) { oss << (insert_sep++ ? sep.c_str() : ""); oss << i; } return oss.str(); }; } // namespace BuildMaps::Base #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/000077500000000000000000000000001516554100600272265ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/TARGETS000066400000000000000000000036401516554100600302650ustar00rootroot00000000000000{ "linked_map": { "type": ["@", "rules", "CC", "library"] , "name": ["linked_map"] , "hdrs": ["linked_map.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/multithreading", "atomic_value"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "buildtool", "build_engine", "expression"] } , "expression_ptr_interface": { "type": ["@", "rules", "CC", "library"] , "name": ["expression_ptr_interface"] , "hdrs": ["expression_ptr.hpp", "function_map.hpp"] , "deps": [ "linked_map" , ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "build_engine", "expression"] } , "expression": { "type": ["@", "rules", "CC", "library"] , "name": ["expression"] , "hdrs": [ "configuration.hpp" , "expression.hpp" , "evaluator.hpp" , "target_result.hpp" , "target_node.hpp" ] , "srcs": [ "expression_ptr.cpp" , "expression.cpp" , "evaluator.cpp" , "target_result.cpp" , "target_node.cpp" ] , "deps": [ "expression_ptr_interface" , "linked_map" , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/multithreading", "atomic_value"] , ["src/utils/cpp", "concepts"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "build_engine", "expression"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "json"] , ["src/utils/cpp", "path"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/configuration.hpp000066400000000000000000000147511516554100600326160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP #include #include #include #include #include #include #include #include // std::move #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/utils/cpp/concepts.hpp" #include "src/utils/cpp/gsl.hpp" // Decorator for Expression containing a map. Adds Prune() and Update(). class Configuration { public: explicit Configuration(ExpressionPtr expr) noexcept : expr_{std::move(expr)} { ExpectsAudit(expr_->IsMap()); } explicit Configuration(Expression::map_t&& map) noexcept : expr_{ExpressionPtr{std::move(map)}} {} Configuration() noexcept = default; ~Configuration() noexcept = default; Configuration(Configuration const&) noexcept = default; Configuration(Configuration&&) noexcept = default; auto operator=(Configuration const&) noexcept -> Configuration& = default; auto operator=(Configuration&&) noexcept -> Configuration& = default; [[nodiscard]] auto operator[](std::string const& key) const -> ExpressionPtr { return expr_->Get(key, Expression::none_t{}); } [[nodiscard]] auto operator[](ExpressionPtr const& key) const -> ExpressionPtr { return expr_->Get(key->String(), Expression::none_t{}); } [[nodiscard]] auto ToString() const -> std::string { return expr_->ToString(); } [[nodiscard]] auto ToJson() const -> nlohmann::json { return expr_->ToJson(); } [[nodiscard]] auto Enumerate(const std::string& prefix, std::size_t width) const -> std::string { std::stringstream ss{}; if (width > prefix.size()) { std::size_t actual_width = width - prefix.size(); for (auto const& [key, value] : expr_->Map()) { std::string key_str = Expression{key}.ToString(); if (actual_width > key_str.size() + 3) { ss << prefix << key_str << " : "; std::size_t remain = actual_width - key_str.size() - 3; std::string val_str = value->ToAbbrevString(remain); if (val_str.size() >= remain) { ss << val_str.substr(0, remain - 3) << "..."; } else { ss << val_str; } } else { ss << prefix << key_str.substr(0, actual_width); } ss << std::endl; } } return ss.str(); } [[nodiscard]] auto operator==(const Configuration& other) const -> bool { return expr_ == other.expr_; } [[nodiscard]] auto operator<(const Configuration& other) const -> bool { return expr_->ToHash() < other.expr_->ToHash(); } [[nodiscard]] auto hash() const noexcept -> std::size_t { return std::hash{}(expr_); } template [[nodiscard]] auto Prune(T const& vars) const -> Configuration { auto subset = Expression::map_t::underlying_map_t{}; std::for_each(vars.begin(), vars.end(), [&](auto const& k) { auto const& map = expr_->Map(); auto v = map.Find(k); if (v) { subset.emplace(k, **v); } else { subset.emplace(k, Expression::kNone); } }); return Configuration{Expression::map_t{subset}}; } [[nodiscard]] auto Prune(ExpressionPtr const& vars) const -> Configuration { auto subset = Expression::map_t::underlying_map_t{}; auto const& list = vars->List(); std::for_each(list.begin(), list.end(), [&](auto const& k) { auto const& map = expr_->Map(); auto const key = k->String(); auto v = map.Find(key); if (v) { subset.emplace(key, **v); } else { subset.emplace(key, ExpressionPtr{Expression::none_t{}}); } }); return Configuration{Expression::map_t{subset}}; } template requires(Expression::IsValidType() or std::is_same_v) [[nodiscard]] auto Update(std::string const& name, T const& value) const -> Configuration { auto update = Expression::map_t::underlying_map_t{}; update.emplace(name, value); return Configuration{Expression::map_t{expr_, update}}; } [[nodiscard]] auto Update( Expression::map_t::underlying_map_t const& map) const -> Configuration { if (map.empty()) { return *this; } return Configuration{Expression::map_t{expr_, map}}; } [[nodiscard]] auto Update(ExpressionPtr const& map) const -> Configuration { ExpectsAudit(map->IsMap()); if (map->Map().empty()) { return *this; } return Configuration{Expression::map_t{expr_, map}}; } [[nodiscard]] auto VariableFixed(std::string const& x) const -> bool { return expr_->Map().Find(x).has_value(); } [[nodiscard]] auto Expr() const& -> ExpressionPtr const& { return expr_; } [[nodiscard]] auto Expr() && -> ExpressionPtr { return std::move(expr_); } private: ExpressionPtr expr_{Expression::kEmptyMap}; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(Configuration const& p) const noexcept -> std::size_t { return p.hash(); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/evaluator.cpp000066400000000000000000001621241516554100600317420ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/evaluator.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include // std::move #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/utils/cpp/path.hpp" namespace { using namespace std::string_literals; using number_t = Expression::number_t; using list_t = Expression::list_t; using map_t = Expression::map_t; auto ValueIsTrue(ExpressionPtr const& val) -> bool { if (val->IsNone()) { return false; } if (val->IsBool()) { return *val != false; } if (val->IsNumber()) { return *val != number_t{0}; } if (val->IsString()) { return *val != ""s; } if (val->IsList()) { return not val->List().empty(); } if (val->IsMap()) { return not val->Map().empty(); } return true; } auto Flatten(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Flatten expects list but instead got: {}.", expr->ToString())}; } if (expr->List().empty()) { return expr; } auto const& list = expr->List(); std::size_t size{}; std::for_each(list.begin(), list.end(), [&](auto const& l) { if (not l->IsList()) { throw Evaluator::EvaluationError{ fmt::format("Non-list entry found for argument in flatten: {}.", l->ToString())}; } size += l->List().size(); }); auto result = Expression::list_t{}; result.reserve(size); std::for_each(list.begin(), list.end(), [&](auto const& l) { std::copy( l->List().begin(), l->List().end(), std::back_inserter(result)); }); return ExpressionPtr{result}; } auto Addition(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Addition expects a list, but found: {}", expr->ToString())}; } Expression::number_t result = 0.0; auto const& list = expr->List(); std::for_each(list.begin(), list.end(), [&](auto const& f) { if (not f->IsNumber()) { throw Evaluator::EvaluationError{fmt::format( "Non-number entry found for argument to addition: {}", f->ToString())}; } result += f->Number(); }); return ExpressionPtr(result); } auto Multiplication(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Multiplication expects a list, but found: {}", expr->ToString())}; } Expression::number_t result = 1.0; auto const& list = expr->List(); std::for_each(list.begin(), list.end(), [&](auto const& f) { if (not f->IsNumber()) { throw Evaluator::EvaluationError{fmt::format( "Non-number entry found for argument to multiplication: {}", f->ToString())}; } result *= f->Number(); }); return ExpressionPtr(result); } auto All(ExpressionPtr const& list) -> ExpressionPtr { for (auto const& c : list->List()) { if (not ValueIsTrue(c)) { return ExpressionPtr{false}; } } return ExpressionPtr{true}; } auto Any(ExpressionPtr const& list) -> ExpressionPtr { for (auto const& c : list->List()) { if (ValueIsTrue(c)) { return ExpressionPtr{true}; } } return ExpressionPtr{false}; } // logical AND with short-circuit evaluation auto LogicalAnd(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const list = expr->At("$1")) { auto const& l = list->get(); if (not l->IsList()) { throw Evaluator::EvaluationError{ fmt::format("Non-list entry found for argument in and: {}.", l->ToString())}; } for (auto const& c : l->List()) { if (not ValueIsTrue(eval(c, env))) { return ExpressionPtr{false}; } } } return ExpressionPtr{true}; } // logical OR with short-circuit evaluation auto LogicalOr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const list = expr->At("$1")) { auto const& l = list->get(); if (not l->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Non-list entry found for argument in or: {}.", l->ToString())}; } for (auto const& c : l->List()) { if (ValueIsTrue(eval(c, env))) { return ExpressionPtr{true}; } } } return ExpressionPtr{false}; } // Logical Negation auto Not(ExpressionPtr const& expr) -> ExpressionPtr { if (ValueIsTrue(expr)) { return ExpressionPtr{false}; } return ExpressionPtr{true}; } auto Keys(ExpressionPtr const& d) -> ExpressionPtr { auto const& m = d->Map(); auto result = Expression::list_t{}; result.reserve(m.size()); std::for_each(m.begin(), m.end(), [&](auto const& item) { result.emplace_back(ExpressionPtr{item.first}); }); return ExpressionPtr{result}; } auto Values(ExpressionPtr const& d) -> ExpressionPtr { return ExpressionPtr{d->Map().Values()}; } auto Enumerate(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "enumerate expects list but instead got: {}.", expr->ToString())}; } auto result = Expression::map_t::underlying_map_t{}; std::size_t count = 0; for (auto const& entry : expr->List()) { result[fmt::format("{:010d}", count)] = entry; count++; } return ExpressionPtr{Expression::map_t{result}}; } auto Set(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{ fmt::format("set expects list of strings but instead got: {}.", expr->ToString())}; } auto result = Expression::map_t::underlying_map_t{}; for (auto const& entry : expr->List()) { if (not entry->IsString()) { throw Evaluator::EvaluationError{ fmt::format("set expects list of strings found entry: {}.", entry->ToString())}; } result[entry->String()] = Expression::kTrue; } return ExpressionPtr{Expression::map_t{result}}; } auto Reverse(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "reverse expects list but instead got: {}.", expr->ToString())}; } auto reverse_result = Expression::list_t(expr->List()); std::reverse(reverse_result.begin(), reverse_result.end()); return ExpressionPtr{reverse_result}; } auto Length(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "length expects list but instead got: {}.", expr->ToString())}; } return ExpressionPtr{ static_cast(expr->List().size())}; } auto NubRight(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "nub_right expects list but instead got: {}.", expr->ToString())}; } if (not expr->IsCacheable()) { throw Evaluator::EvaluationError{ fmt::format("Implicit comparison by passing name-containing value " "to nub_right: {}", expr->ToString())}; } // short-cut evaluation for efficiency if (expr->List().empty()) { return expr; } auto const& list = expr->List(); auto reverse_result = Expression::list_t{}; reverse_result.reserve(list.size()); auto seen = std::unordered_set{}; seen.reserve(list.size()); std::for_each(list.rbegin(), list.rend(), [&](auto const& l) { if (not seen.contains(l)) { reverse_result.push_back(l); seen.insert(l); } }); std::reverse(reverse_result.begin(), reverse_result.end()); return ExpressionPtr{reverse_result}; } auto NubLeft(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "nub_left expects list but instead got: {}.", expr->ToString())}; } if (not expr->IsCacheable()) { throw Evaluator::EvaluationError{ fmt::format("Implicit comparison by passing name-containing value " "to nub_left: {}", expr->ToString())}; } // short-cut evaluation for efficiency if (expr->List().empty()) { return expr; } auto const& list = expr->List(); auto result = Expression::list_t{}; result.reserve(list.size()); auto seen = std::unordered_set{}; seen.reserve(list.size()); std::for_each(list.begin(), list.end(), [&](auto const& l) { if (not seen.contains(l)) { result.push_back(l); seen.insert(l); } }); return ExpressionPtr{result}; } auto Range(ExpressionPtr const& expr) -> ExpressionPtr { std::size_t len = 0; if (expr->IsNumber() && expr->Number() > 0.0) { len = static_cast(std::lround(expr->Number())); } if (expr->IsString()) { len = static_cast(std::atol(expr->String().c_str())); } auto result = Expression::list_t{}; result.reserve(len); for (std::size_t i = 0; i < len; i++) { result.emplace_back(fmt::format("{}", i)); } return ExpressionPtr{result}; } auto ChangeEndingTo(ExpressionPtr const& name, ExpressionPtr const& ending) -> ExpressionPtr { std::filesystem::path path{name->String()}; return ExpressionPtr{(path.parent_path() / path.stem()).string() + ending->String()}; } auto BaseName(ExpressionPtr const& name) -> ExpressionPtr { std::filesystem::path path{name->String()}; return ExpressionPtr{path.filename().string()}; } auto ShellQuote(std::string arg) -> std::string { auto start_pos = size_t{}; std::string from{"'"}; std::string to{"'\\''"}; while ((start_pos = arg.find(from, start_pos)) != std::string::npos) { arg.replace(start_pos, from.length(), to); start_pos += to.length(); } return fmt::format("'{}'", arg); } template auto Join(ExpressionPtr const& expr, std::string const& sep) -> ExpressionPtr { if constexpr (kAllowString) { if (expr->IsString()) { auto string = expr->String(); if constexpr (kDoQuote) { string = ShellQuote(std::move(string)); } return ExpressionPtr{std::move(string)}; } } if (expr->IsList()) { auto const& list = expr->List(); int insert_sep{}; std::stringstream ss{}; std::for_each(list.begin(), list.end(), [&](auto const& e) { ss << (insert_sep++ ? sep : ""); auto string = e->String(); if constexpr (kDoQuote) { string = ShellQuote(std::move(string)); } ss << std::move(string); }); return ExpressionPtr{ss.str()}; } throw Evaluator::EvaluationError{ fmt::format("Join expects a list of strings{}, but got: {}.", kAllowString ? " or a single string" : "", expr->ToString())}; } template auto Union(Expression::list_t const& dicts, std::size_t from, std::size_t to) -> ExpressionPtr { if (to <= from) { return Expression::kEmptyMap; } if (to == from + 1) { auto const& entry = dicts[from]; if (not entry->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "Map union list element is not a map: {}", entry->ToString())}; } return entry; } std::size_t mid = from + (to - from) / 2; auto left = Union(dicts, from, mid); auto right = Union(dicts, mid, to); if (left->Map().empty()) { return right; } if (right->Map().empty()) { return left; } if constexpr (kDisjoint) { auto dup = left->Map().FindConflictingDuplicate(right->Map()); if (dup) { auto left_val = left->Get(dup->get(), Expression::none_t{}); auto right_val = right->Get(dup->get(), Expression::none_t{}); throw Evaluator::EvaluationError{ fmt::format("Map union not essentially disjoint as claimed, " "duplicate key {}; conflicting values:\n- {}\n- {}", nlohmann::json(dup->get()).dump(), left_val->ToAbbrevString( Evaluator::GetExpressionLogLimit()), right_val->ToAbbrevString( Evaluator::GetExpressionLogLimit())), false, false, std::vector{left_val, right_val}}; } } return ExpressionPtr{Expression::map_t{left, right}}; } template auto Union(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Union expects list of maps but got: {}.", expr->ToString())}; } auto const& list = expr->List(); if (list.empty()) { return Expression::kEmptyMap; } return Union(list, 0, list.size()); } auto ConcatTargetName(ExpressionPtr const& expr, ExpressionPtr const& append) -> ExpressionPtr { if (expr->IsString()) { return ExpressionPtr{expr->String() + append->String()}; } if (expr->IsList()) { auto list = Expression::list_t{}; auto not_last = expr->List().size(); bool all_string = true; std::for_each( expr->List().begin(), expr->List().end(), [&](auto const& e) { all_string = all_string and e->IsString(); if (all_string) { list.emplace_back(ExpressionPtr{ e->String() + (--not_last ? "" : append->String())}); } }); if (all_string) { return ExpressionPtr{list}; } } throw Evaluator::EvaluationError{fmt::format( "Unsupported expression for concat: {}.", expr->ToString())}; } auto EvalArgument(ExpressionPtr const& expr, std::string const& argument, const SubExprEvaluator& eval, Configuration const& env) -> ExpressionPtr { try { return eval(expr[argument], env); } catch (Evaluator::EvaluationError const& ex) { throw Evaluator::EvaluationError::WhileEval( fmt::format("Evaluating argument {}:", argument), ex); } catch (std::exception const& ex) { throw Evaluator::EvaluationError::WhileEvaluating( fmt::format("Evaluating argument {}:", argument), ex); } } auto UnaryExpr(std::function const& f) -> std::function { return [f](auto&& eval, auto const& expr, auto const& env) { auto argument = EvalArgument(expr, "$1", eval, env); try { return f(argument); } catch (Evaluator::EvaluationError const& ex) { throw Evaluator::EvaluationError::WhileEval( fmt::format("Having evaluated the argument to {}:", argument->ToAbbrevString( Evaluator::GetExpressionLogLimit())), ex); } catch (std::exception const& ex) { throw Evaluator::EvaluationError::WhileEvaluating( fmt::format("Having evaluated the argument to {}:", argument->ToAbbrevString( Evaluator::GetExpressionLogLimit())), ex); } }; } auto AndExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const conds = expr->At("$1")) { return conds->get()->IsList() ? LogicalAnd(std::move(eval), expr, env) : UnaryExpr(All)(std::move(eval), expr, env); } return ExpressionPtr{true}; } auto OrExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const conds = expr->At("$1")) { return conds->get()->IsList() ? LogicalOr(std::move(eval), expr, env) : UnaryExpr(Any)(std::move(eval), expr, env); } return ExpressionPtr{false}; } auto VarExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto result = env[expr["name"]]; if (result->IsNone()) { return eval(expr->Get("default", Expression::none_t{}), env); } return result; } auto OnlyInQuasiQuote(SubExprEvaluator&& /*eval*/, ExpressionPtr const& /*expr*/, Configuration const& /*env*/) -> ExpressionPtr { throw Evaluator::EvaluationError{ R"("," and ",@" are only evaluated within quasi-quote ("`") environments.)"}; } auto ExpandQuasiQuote(const SubExprEvaluator& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr; auto ExpandQuasiQuoteListEntry(const SubExprEvaluator& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (expr->IsList()) { auto result = Expression::list_t{}; for (auto const& entry : expr->List()) { auto expanded = ExpandQuasiQuoteListEntry(eval, entry, env); std::copy(expanded->List().begin(), expanded->List().end(), std::back_inserter(result)); } return ExpressionPtr{Expression::list_t{ExpressionPtr{result}}}; } if (expr->IsMap()) { if (auto type_token = expr->At("type")) { if (type_token->get()->IsString()) { auto token = type_token->get()->String(); if (token == ",") { auto arg = expr->At("$1"); if (not arg) { return ExpressionPtr{Expression::kNone}; } auto result = eval(*arg, env); return ExpressionPtr{Expression::list_t{result}}; } if (token == ",@") { auto arg = expr->At("$1"); if (not arg) { return Expression::kEmptyList; } auto result = eval(*arg, env); if (not result->IsList()) { throw Evaluator::EvaluationError{fmt::format( "Argument of \",@\"-expresion {} should evaluate " "to a list, but obtained {}", expr->ToString(), result->ToString())}; } return result; } } } // regular map, walk through the entries and quasiquote expand them auto result = Expression::map_t::underlying_map_t{}; for (auto const& el : expr->Map()) { auto expanded = ExpandQuasiQuote(eval, el.second, env); result[el.first] = expanded; } return ExpressionPtr{Expression::map_t{result}}; } // not a container, spliced literally, i.e., return singleton list return ExpressionPtr{Expression::list_t{expr}}; } auto ExpandQuasiQuote(const SubExprEvaluator& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (expr->IsList()) { auto result = Expression::list_t{}; for (auto const& entry : expr->List()) { auto expanded = ExpandQuasiQuoteListEntry(eval, entry, env); std::copy(expanded->List().begin(), expanded->List().end(), std::back_inserter(result)); } return ExpressionPtr{result}; } if (expr->IsMap()) { if (auto type_token = expr->At("type")) { if (type_token->get()->IsString()) { auto token = type_token->get()->String(); if (token == ",") { auto arg = expr->At("$1"); if (not arg) { return Expression::kNone; } return eval(*arg, env); } if (token == ",@") { throw Evaluator::EvaluationError{fmt::format( "\",@\"-expression found in non-list context: {}", expr->ToString())}; } } } // regular map, walk through the entries and quasiquote expand them auto result = Expression::map_t::underlying_map_t{}; for (auto const& el : expr->Map()) { auto expanded = ExpandQuasiQuote(eval, el.second, env); result[el.first] = expanded; } return ExpressionPtr{Expression::map_t{result}}; } // not a container, return literal anyway return expr; } auto QuasiQuoteExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const to_expand = expr->At("$1")) { return ExpandQuasiQuote(eval, *to_expand, env); } return Expression::kNone; } auto QuoteExpr(SubExprEvaluator&& /*eval*/, ExpressionPtr const& expr, Configuration const& /*env*/) -> ExpressionPtr { if (auto const literal = expr->At("$1")) { return *literal; } return Expression::kNone; } auto IfExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (ValueIsTrue(EvalArgument(expr, "cond", eval, env))) { return eval(expr->Get("then", list_t{}), env); } return eval(expr->Get("else", list_t{}), env); } auto CondExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& cond = expr->At("cond"); if (cond) { if (not cond->get()->IsList()) { throw Evaluator::EvaluationError{fmt::format( "cond in cond has to be a list of pairs, but found {}", cond->get()->ToString())}; } for (const auto& pair : cond->get()->List()) { if (not pair->IsList() or pair->List().size() != 2) { throw Evaluator::EvaluationError{ fmt::format("cond in cond has to be a list of pairs, " "but found entry {}", pair->ToString())}; } if (ValueIsTrue(eval(pair->List()[0], env))) { return eval(pair->List()[1], env); } } } return eval(expr->Get("default", list_t{}), env); } auto CaseExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& cases = expr->At("case"); if (cases) { if (not cases->get()->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "case in case has to be a map of expressions, but found {}", cases->get()->ToString())}; } auto const& e = expr->At("expr"); if (not e) { throw Evaluator::EvaluationError{"missing expr in case"}; } auto const& key = eval(e->get(), env); if (not key->IsString()) { throw Evaluator::EvaluationError{fmt::format( "expr in case must evaluate to string, but found {}", key->ToString())}; } if (auto const& val = cases->get()->At(key->String())) { return eval(val->get(), env); } } return eval(expr->Get("default", list_t{}), env); } auto SeqCaseExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& cases = expr->At("case"); if (cases) { if (not cases->get()->IsList()) { throw Evaluator::EvaluationError{fmt::format( "case in case* has to be a list of pairs, but found {}", cases->get()->ToString())}; } auto const& e = expr->At("expr"); if (not e) { throw Evaluator::EvaluationError{"missing expr in case"}; } auto const& cmp = eval(e->get(), env); if (not cmp->IsCacheable()) { throw Evaluator::EvaluationError{fmt::format( "Comparison of name-containing values: {}", cmp->ToString())}; } for (const auto& pair : cases->get()->List()) { if (not pair->IsList() or pair->List().size() != 2) { throw Evaluator::EvaluationError{ fmt::format("case in case* has to be a list of pairs, " "but found entry {}", pair->ToString())}; } if (cmp == eval(pair->List()[0], env)) { return eval(pair->List()[1], env); } } } return eval(expr->Get("default", list_t{}), env); } auto EqualExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto a = EvalArgument(expr, "$1", eval, env); if (not a->IsCacheable()) { throw Evaluator::EvaluationError{fmt::format( "Comparison of name-containing values; first argument is {}", a->ToString())}; } auto b = EvalArgument(expr, "$2", eval, env); if (not b->IsCacheable()) { throw Evaluator::EvaluationError{fmt::format( "Comparison of name-containing values; second argument is {}", b->ToString())}; } return ExpressionPtr{a == b}; } auto ChangeEndingExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto name = eval(expr->Get("$1", ""s), env); auto ending = eval(expr->Get("ending", ""s), env); return ChangeEndingTo(name, ending); } auto JoinExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto list = eval(expr->Get("$1", list_t{}), env); auto separator = eval(expr->Get("separator", ""s), env); return Join(list, separator->String()); } auto JoinCmdExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& list = eval(expr->Get("$1", list_t{}), env); return Join(list, " "); } auto JsonEncodeExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& value = eval(expr->Get("$1", list_t{}), env); return ExpressionPtr{ value->ToJson(Expression::JsonMode::NullForNonJson).dump()}; } auto EscapeCharsExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto string = eval(expr->Get("$1", ""s), env); auto chars = eval(expr->Get("chars", ""s), env); auto escape_prefix = eval(expr->Get("escape_prefix", "\\"s), env); std::stringstream ss{}; std::for_each( string->String().begin(), string->String().end(), [&](auto const& c) { auto do_escape = chars->String().find(c) != std::string::npos; ss << (do_escape ? escape_prefix->String() : "") << c; }); return ExpressionPtr{ss.str()}; } auto LookupExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto k = eval(expr["key"], env); auto d = eval(expr["map"], env); if (not k->IsString()) { throw Evaluator::EvaluationError{fmt::format( "Key expected to be string but found {}.", k->ToString())}; } if (not d->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "Map expected to be mapping but found {}.", d->ToString())}; } auto const& lookup = d->Map().Find(k->String()); if (lookup and (*lookup)->IsNotNull()) { return **lookup; } return eval(expr->Get("default", Expression::none_t()), env); } auto ArrayAccessExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto k = eval(expr["index"], env); auto d = eval(expr["list"], env); if (not d->IsList()) { throw Evaluator::EvaluationError{fmt::format( "List expected to be list, but found {}.", d->ToString())}; } auto len = static_cast(d->List().size()); int64_t index = 0; if (k->IsNumber()) { index = static_cast(std::lround(k->Number())); } if (k->IsString()) { index = std::atol(k->String().c_str()); } if (0 <= index and index < len) { return d->List()[static_cast(index)]; } if (index < 0 and len + index >= 0) { return d->List()[static_cast(len + index)]; } return eval(expr->Get("default", Expression::none_t()), env); } auto EmptyMapExpr(SubExprEvaluator&& /*eval*/, ExpressionPtr const& /*expr*/, Configuration const& /*env*/) -> ExpressionPtr { return Expression::kEmptyMap; } auto SingletonMapExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto key = EvalArgument(expr, "key", eval, env); auto value = EvalArgument(expr, "value", eval, env); return ExpressionPtr{Expression::map_t{key->String(), value}}; } auto ToSubdirExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto d = eval(expr["$1"], env); auto s = eval(expr->Get("subdir", "."s), env); auto flat = ValueIsTrue(eval(expr->Get("flat", false), env)); std::filesystem::path subdir{s->String()}; auto result = Expression::map_t::underlying_map_t{}; if (flat) { for (auto const& el : d->Map()) { std::filesystem::path k{el.first}; auto new_key = ToNormalPath(subdir / k.filename()).string(); if (result.contains(new_key) and not((result[new_key] == el.second) and el.second->IsCacheable())) { // Check if the user specifed an error message for that case, // otherwise just generate a generic error message. auto msg_expr = expr->Map().Find("msg"); if (not msg_expr) { throw Evaluator::EvaluationError{fmt::format( "Flat staging of {} to subdir {} conflicts on path {}", d->ToString(), subdir.string(), new_key)}; } std::string msg; try { auto msg_val = eval(**msg_expr, env); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + (**msg_expr)->ToString(); } std::stringstream ss{}; ss << msg << std::endl; ss << "Reason: flat staging to subdir " << subdir.string() << " conflicts on path " << new_key << std::endl; ss << "Map to flatly stage was " << d->ToString() << std::endl; throw Evaluator::EvaluationError(ss.str(), false, true); } result[new_key] = el.second; } } else { for (auto const& el : d->Map()) { auto new_key = ToNormalPath(subdir / el.first).string(); if (auto const it = result.find(new_key); it != result.end() and (not((it->second == el.second) and el.second->IsCacheable()))) { auto msg_expr = expr->Map().Find("msg"); if (not msg_expr) { throw Evaluator::EvaluationError{fmt::format( "Staging of {} to subdir {} conflicts on path {}", d->ToString(), subdir.string(), new_key)}; } std::string msg; try { auto msg_val = eval(**msg_expr, env); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + (**msg_expr)->ToString(); } std::stringstream ss{}; ss << msg << std::endl; ss << "Reason: staging to subdir " << subdir.string() << " conflicts on new path " << new_key << std::endl; ss << "Map to stage was " << d->ToString() << std::endl; throw Evaluator::EvaluationError(ss.str(), false, true); } result.emplace(std::move(new_key), el.second); } } return ExpressionPtr{Expression::map_t{result}}; } auto FromSubdirExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto d = eval(expr["$1"], env); auto s = eval(expr->Get("subdir", "."s), env); std::filesystem::path subdir{s->String()}; auto result = Expression::map_t::underlying_map_t{}; for (auto const& el : d->Map()) { auto new_path = ToNormalPath( std::filesystem::path(el.first).lexically_relative(subdir)); if (PathIsNonUpwards(new_path)) { auto new_key = new_path.string(); if (auto const it = result.find(new_key); it != result.end() && (!((it->second == el.second) && el.second->IsCacheable()))) { throw Evaluator::EvaluationError{ fmt::format("Staging conflict for path {}", nlohmann::json(new_key).dump())}; } result.emplace(std::move(new_key), el.second); } } return ExpressionPtr{Expression::map_t{result}}; } auto ForeachExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto range_list = eval(expr->Get("range", list_t{}), env); if (range_list->List().empty()) { return Expression::kEmptyList; } auto const& var = expr->Get("var", "_"s); auto const& body = expr->Get("body", list_t{}); auto result = Expression::list_t{}; result.reserve(range_list->List().size()); std::transform(range_list->List().begin(), range_list->List().end(), std::back_inserter(result), [&](auto const& x) { return eval(body, env.Update(var->String(), x)); }); return ExpressionPtr{result}; } auto ForeachMapExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto range_map = eval(expr->Get("range", Expression::kEmptyMapExpr), env); if (range_map->Map().empty()) { return Expression::kEmptyList; } auto const& var = expr->Get("var_key", "_"s); auto const& var_val = expr->Get("var_val", "$_"s); auto const& body = expr->Get("body", list_t{}); auto result = Expression::list_t{}; result.reserve(range_map->Map().size()); std::transform(range_map->Map().begin(), range_map->Map().end(), std::back_inserter(result), [&](auto const& it) { return eval(body, env.Update(var->String(), it.first) .Update(var_val->String(), it.second)); }); return ExpressionPtr{result}; } auto ZipWithExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto range_1 = eval(expr->Get("range_1", Expression::kEmptyList), env); auto const& range_1_list = range_1->List(); if (range_1_list.empty()) { return Expression::kEmptyList; } auto range_2 = eval(expr->Get("range_2", Expression::kEmptyList), env); auto const& range_2_list = range_2->List(); if (range_2_list.empty()) { return Expression::kEmptyList; } auto const size = std::min(range_1_list.size(), range_2_list.size()); auto const& var_1 = expr->Get("var_1", "$1"s); auto const& var_2 = expr->Get("var_2", "$2"s); auto const& body = expr->Get("body", list_t{}); auto result = Expression::list_t{}; result.reserve(size); for (auto it = std::make_pair(range_1_list.begin(), range_2_list.begin()); it.first != range_1_list.end() and it.second != range_2_list.end(); ++it.first, ++it.second) { result.emplace_back(eval(body, env.Update(var_1->String(), *it.first) .Update(var_2->String(), *it.second))); } return ExpressionPtr{result}; } auto ZipMapExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto range_key = eval(expr->Get("range_key", Expression::kEmptyList), env); auto const& range_key_list = range_key->List(); if (range_key_list.empty()) { return Expression::kEmptyMap; } auto range_val = eval(expr->Get("range_val", Expression::kEmptyList), env); auto const& range_val_list = range_val->List(); if (range_val_list.empty()) { return Expression::kEmptyMap; } auto result = Expression::map_t::underlying_map_t{}; for (auto it = std::make_pair(range_key_list.begin(), range_val_list.begin()); it.first != range_key_list.end() and it.second != range_val_list.end(); ++it.first, ++it.second) { result[(*it.first)->String()] = *it.second; } return ExpressionPtr{Expression::map_t{result}}; } auto FoldLeftExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& var = expr->Get("var", "_"s); auto const& accum_var = expr->Get("accum_var", "$1"s); auto range_list = eval(expr["range"], env); auto val = eval(expr->Get("start", list_t{}), env); auto const& body = expr->Get("body", list_t{}); for (auto const& x : range_list->List()) { val = eval( body, env.Update({{var->String(), x}, {accum_var->String(), val}})); } return val; } auto LetExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto const& bindings = expr->At("bindings"); auto new_env = env; if (bindings) { if (not bindings->get()->IsList()) { throw Evaluator::EvaluationError{fmt::format( "bindings in let* has to be a list of pairs, but found {}", bindings->get()->ToString())}; } int pos = -1; for (const auto& binding : bindings->get()->List()) { ++pos; if (not binding->IsList() or binding->List().size() != 2) { throw Evaluator::EvaluationError{ fmt::format("bindings in let* has to be a list of pairs, " "but found entry {}", binding->ToString())}; } auto const& x_exp = binding[0]; if (not x_exp->IsString()) { throw Evaluator::EvaluationError{ fmt::format("variable names in let* have to be strings, " "but found binding entry {}", binding->ToString())}; } ExpressionPtr val; try { val = eval(binding[1], new_env); } catch (Evaluator::EvaluationError const& ex) { throw Evaluator::EvaluationError::WhileEval( fmt::format("Evaluating entry {} in bindings, binding {}:", pos, x_exp->ToString()), ex); } catch (std::exception const& ex) { throw Evaluator::EvaluationError::WhileEvaluating( fmt::format("Evaluating entry {} in bindings, binding {}:", pos, x_exp->ToString()), ex); } new_env = new_env.Update(x_exp->String(), val); } } auto const& body = expr->Get("body", map_t{}); try { return eval(body, new_env); } catch (Evaluator::EvaluationError const& ex) { throw Evaluator::EvaluationError::WhileEval("Evaluating the body:", ex); } catch (std::exception const& ex) { throw Evaluator::EvaluationError::WhileEvaluating( "Evaluating the body:", ex); } } auto EnvExpr(SubExprEvaluator&& /*eval*/, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { if (auto const& vars = expr->At("vars")) { if (not vars->get()->IsList()) { throw Evaluator::EvaluationError{fmt::format( "vars in env has to be a list of strings, but found {}", vars->get()->ToString())}; } for (const auto& var : vars->get()->List()) { if (not var->IsString()) { throw Evaluator::EvaluationError{ fmt::format("vars in env has to be a list of strings, " "but found entry {}", var->ToString())}; } } return env.Prune(vars->get()).Expr(); } return Expression::kEmptyMap; } auto ConcatTargetNameExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto p1 = eval(expr->Get("$1", ""s), env); auto p2 = eval(expr->Get("$2", ""s), env); return ConcatTargetName( p1, Join(p2, "")); } auto ContextExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { try { return eval(expr->Get("$1", Expression::kNone), env); } catch (Evaluator::EvaluationError const& ex) { auto msg_expr = expr->Get("msg", map_t{}); std::string context{}; try { auto msg_val = eval(msg_expr, env); context = msg_val->ToString(); } catch (std::exception const&) { context = "[non evaluating term] " + msg_expr->ToString(); } std::stringstream ss{}; ss << "In Context " << context << std::endl; ss << ex.what(); throw Evaluator::EvaluationError( ss.str(), true, true, ex.InvolvedObjects()); } catch (std::exception const& ex) { auto msg_expr = expr->Get("msg", map_t{}); std::string context{}; try { auto msg_val = eval(msg_expr, env); context = msg_val->ToString(); } catch (std::exception const&) { context = "[non evaluating term] " + msg_expr->ToString(); } std::stringstream ss{}; ss << "In Context " << context << std::endl; ss << ex.what(); throw Evaluator::EvaluationError(ss.str(), true, true); } } auto DisjointUnionExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto argument = EvalArgument(expr, "$1", eval, env); if (not argument->IsCacheable()) { throw Evaluator::EvaluationError{ fmt::format("Argument to disjoint_map_union is name-containing: {}", argument->ToString())}; } try { return Union(argument); } catch (Evaluator::EvaluationError const& ex) { auto msg_expr = expr->Map().Find("msg"); if (not msg_expr) { throw Evaluator::EvaluationError::WhileEval( fmt::format("Having evaluated the argument to {}:", argument->ToAbbrevString( Evaluator::GetExpressionLogLimit())), ex); } std::string msg; try { auto msg_val = eval(**msg_expr, env); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + (**msg_expr)->ToString(); } std::stringstream ss{}; ss << msg << std::endl; ss << "Underlying " << ex.what() << std::endl; ss << "The argument of the union was " << argument->ToAbbrevString(Evaluator::GetExpressionLogLimit()); throw Evaluator::EvaluationError( ss.str(), false, true, ex.InvolvedObjects()); } catch (std::exception const& ex) { auto msg_expr = expr->Map().Find("msg"); if (not msg_expr) { throw Evaluator::EvaluationError::WhileEvaluating( fmt::format("Having evaluated the argument to {}:", argument->ToAbbrevString( Evaluator::GetExpressionLogLimit())), ex); } std::string msg; try { auto msg_val = eval(**msg_expr, env); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + (**msg_expr)->ToString(); } std::stringstream ss{}; ss << msg << std::endl; ss << "Reason: " << ex.what() << std::endl; ss << "The argument of the union was " << argument->ToAbbrevString(Evaluator::GetExpressionLogLimit()); throw Evaluator::EvaluationError(ss.str(), false, true); } } auto FailExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto msg = eval(expr->Get("msg", Expression::kNone), env); throw Evaluator::EvaluationError( msg->ToString(), false, /* user error*/ true); } auto AssertExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto val = eval(expr["$1"], env); auto const& var = expr->Get("var", "_"s); auto pred = eval(expr["predicate"], env.Update(var->String(), val)); if (ValueIsTrue(pred)) { return val; } auto msg_expr = expr->Get("msg", Expression::kNone); std::string msg; try { auto msg_val = eval(msg_expr, env.Update(var->String(), val)); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + msg_expr->ToString(); } throw Evaluator::EvaluationError(msg, false, /* user error */ true); } auto AssertNonEmptyExpr(SubExprEvaluator&& eval, ExpressionPtr const& expr, Configuration const& env) -> ExpressionPtr { auto val = eval(expr["$1"], env); if ((val->IsString() and (not val->String().empty())) or (val->IsList() and (not val->List().empty())) or (val->IsMap() and (not val->Map().empty()))) { return val; } auto msg_expr = expr->Get("msg", Expression::kNone); std::string msg; try { auto msg_val = eval(msg_expr, env); msg = msg_val->ToString(); } catch (std::exception const&) { msg = "[non evaluating term] " + msg_expr->ToString(); } std::stringstream ss{}; ss << msg << std::endl; ss << "Expected non-empty value but found: " << val->ToString(); throw Evaluator::EvaluationError(ss.str(), false, true); } auto const kBuiltInFunctions = FunctionMap::MakePtr({{"var", VarExpr}, {"'", QuoteExpr}, {"`", QuasiQuoteExpr}, {",", OnlyInQuasiQuote}, {",@", OnlyInQuasiQuote}, {"if", IfExpr}, {"cond", CondExpr}, {"case", CaseExpr}, {"case*", SeqCaseExpr}, {"fail", FailExpr}, {"assert", AssertExpr}, {"assert_non_empty", AssertNonEmptyExpr}, {"context", ContextExpr}, {"==", EqualExpr}, {"and", AndExpr}, {"or", OrExpr}, {"not", UnaryExpr(Not)}, {"++", UnaryExpr(Flatten)}, {"+", UnaryExpr(Addition)}, {"*", UnaryExpr(Multiplication)}, {"nub_right", UnaryExpr(NubRight)}, {"nub_left", UnaryExpr(NubLeft)}, {"range", UnaryExpr(Range)}, {"change_ending", ChangeEndingExpr}, {"basename", UnaryExpr(BaseName)}, {"join", JoinExpr}, {"join_cmd", JoinCmdExpr}, {"json_encode", JsonEncodeExpr}, {"escape_chars", EscapeCharsExpr}, {"keys", UnaryExpr(Keys)}, {"enumerate", UnaryExpr(Enumerate)}, {"set", UnaryExpr(Set)}, {"reverse", UnaryExpr(Reverse)}, {"length", UnaryExpr(Length)}, {"values", UnaryExpr(Values)}, {"lookup", LookupExpr}, {"[]", ArrayAccessExpr}, {"empty_map", EmptyMapExpr}, {"singleton_map", SingletonMapExpr}, {"disjoint_map_union", DisjointUnionExpr}, {"map_union", UnaryExpr([](auto const& exp) { return Union(exp); })}, {"to_subdir", ToSubdirExpr}, {"from_subdir", FromSubdirExpr}, {"foreach", ForeachExpr}, {"foreach_map", ForeachMapExpr}, {"zip_with", ZipWithExpr}, {"zip_map", ZipMapExpr}, {"foldl", FoldLeftExpr}, {"let*", LetExpr}, {"env", EnvExpr}, {"concat_target_name", ConcatTargetNameExpr}}); auto ExtendedErrorMessage(ExpressionPtr const& expr, Configuration const& env, std::exception const& ex) -> std::string { std::stringstream ss{}; ss << "* "; if (expr->IsMap() and expr->Map().contains("type") and expr["type"]->IsString()) { ss << expr["type"]->ToString() << "-expression "; } ss << expr->ToAbbrevString(Evaluator::GetExpressionLogLimit()) << std::endl; ss << " environment " << std::endl; ss << env.Enumerate(" - ", Evaluator::GetExpressionLogLimit()) << std::endl; ss << ex.what(); return ss.str(); } } // namespace auto Evaluator::EvaluationError::WhileEvaluating(ExpressionPtr const& expr, Configuration const& env, std::exception const& ex) -> Evaluator::EvaluationError { return EvaluationError{ExtendedErrorMessage(expr, env, ex), true /* while_eval */}; } auto Evaluator::EvaluationError::WhileEval(ExpressionPtr const& expr, Configuration const& env, Evaluator::EvaluationError const& ex) -> Evaluator::EvaluationError { if (ex.UserContext()) { return ex; } return EvaluationError{ ExtendedErrorMessage(expr, env, ex), true, false, ex.InvolvedObjects()}; } auto Evaluator::EvaluationError::WhileEvaluating(const std::string& where, std::exception const& ex) -> Evaluator::EvaluationError { std::stringstream ss{}; ss << where << std::endl; ss << ex.what(); return EvaluationError{ss.str(), true /* while_eval */}; } auto Evaluator::EvaluationError::WhileEval(const std::string& where, Evaluator::EvaluationError const& ex) -> Evaluator::EvaluationError { if (ex.UserContext()) { return ex; } std::stringstream ss{}; ss << where << std::endl; ss << ex.what(); return EvaluationError{ss.str(), true, false, ex.InvolvedObjects()}; } auto Evaluator::EvaluateExpression( ExpressionPtr const& expr, Configuration const& env, FunctionMapPtr const& provider_functions, std::function const& logger, std::function const& annotate_object, std::function const& note_user_context) noexcept -> ExpressionPtr { std::stringstream ss{}; try { return Evaluate( expr, env, FunctionMap::MakePtr(kBuiltInFunctions, provider_functions)); } catch (EvaluationError const& ex) { if (ex.UserContext()) { note_user_context(); } else if (ex.WhileEvaluation()) { ss << "Expression evaluation traceback (most recent call last):" << std::endl; } ss << ex.what(); for (auto const& object : ex.InvolvedObjects()) { ss << annotate_object(object); } } catch (std::exception const& ex) { ss << ex.what(); } logger(ss.str()); return ExpressionPtr{nullptr}; } auto Evaluator::Evaluate(ExpressionPtr const& expr, Configuration const& env, FunctionMapPtr const& functions) -> ExpressionPtr { try { if (expr->IsList()) { if (expr->List().empty()) { return expr; } auto list = Expression::list_t{}; std::transform( expr->List().cbegin(), expr->List().cend(), std::back_inserter(list), [&](auto const& e) { return Evaluate(e, env, functions); }); return ExpressionPtr{list}; } if (not expr->IsMap()) { return expr; } if (not expr->Map().contains("type")) { throw EvaluationError{fmt::format( "Object without keyword 'type': {}", expr->ToString())}; } auto const& type = expr["type"]->String(); auto func = functions->Find(type); if (func) { return (**func)( [&functions](auto const& subexpr, auto const& subenv) { return Evaluator::Evaluate(subexpr, subenv, functions); }, expr, env); } throw EvaluationError{ fmt::format("Unknown syntactical construct {}", type)}; } catch (EvaluationError const& ex) { throw EvaluationError::WhileEval(expr, env, ex); } catch (std::exception const& ex) { throw EvaluationError::WhileEvaluating(expr, env, ex); } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/evaluator.hpp000066400000000000000000000114451516554100600317460ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP #include #include #include #include #include #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" class Evaluator { struct ConfigData { std::size_t expression_log_limit{kDefaultExpressionLogLimit}; }; public: /// \brief Set the limit for the size of logging a single expression /// in an error message. static void SetExpressionLogLimit(std::size_t width) { Config().expression_log_limit = width; } static auto GetExpressionLogLimit() -> std::size_t { return Config().expression_log_limit; } class EvaluationError : public std::exception { public: explicit EvaluationError(std::string msg, bool while_eval = false, bool user_context = false, std::vector involved_objetcs = std::vector{}) noexcept : msg_{std::move(msg)}, while_eval_{while_eval}, user_context_{user_context}, involved_objects_{std::move(std::move(involved_objetcs))} { if (not while_eval_) { msg_ = (user_context_ ? "UserError: " : "EvaluationError: ") + msg_; } } [[nodiscard]] auto what() const noexcept -> char const* final { return msg_.c_str(); } [[nodiscard]] auto WhileEvaluation() const -> bool { return while_eval_; } [[nodiscard]] auto UserContext() const -> bool { return user_context_; } [[nodiscard]] auto InvolvedObjects() const& noexcept -> std::vector const& { return involved_objects_; } [[nodiscard]] auto InvolvedObjects() && -> std::vector { return involved_objects_; } [[nodiscard]] static auto WhileEvaluating(ExpressionPtr const& expr, Configuration const& env, std::exception const& ex) -> EvaluationError; [[nodiscard]] static auto WhileEval(ExpressionPtr const& expr, Configuration const& env, EvaluationError const& ex) -> EvaluationError; [[nodiscard]] static auto WhileEvaluating(const std::string& where, std::exception const& ex) -> Evaluator::EvaluationError; [[nodiscard]] static auto WhileEval(const std::string& where, EvaluationError const& ex) -> Evaluator::EvaluationError; private: std::string msg_; bool while_eval_; bool user_context_; std::vector involved_objects_; }; // Exception-free evaluation of expression [[nodiscard]] static auto EvaluateExpression( ExpressionPtr const& expr, Configuration const& env, FunctionMapPtr const& provider_functions, std::function const& logger, std::function const& annotate_object = [](auto const& /*unused*/) { return std::string{}; }, std::function const& note_user_context = []() {}) noexcept -> ExpressionPtr; constexpr static std::size_t kDefaultExpressionLogLimit = 320; private: [[nodiscard]] static auto Evaluate(ExpressionPtr const& expr, Configuration const& env, FunctionMapPtr const& functions) -> ExpressionPtr; [[nodiscard]] static auto Config() noexcept -> ConfigData& { static ConfigData instance{}; return instance; } }; #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/expression.cpp000066400000000000000000000207061516554100600321360ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/expression.hpp" #include #include #include #include #include "fmt/core.h" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/json.hpp" auto Expression::operator[]( std::string const& key) const& -> ExpressionPtr const& { auto value = Map().Find(key); if (value) { return **value; } throw ExpressionTypeError{ fmt::format("Map does not contain key '{}'.", key)}; } auto Expression::operator[](std::string const& key) && -> ExpressionPtr { auto value = std::move(*this).Map().Find(key); if (value) { return std::move(*value); } throw ExpressionTypeError{ fmt::format("Map does not contain key '{}'.", key)}; } auto Expression::operator[]( ExpressionPtr const& key) const& -> ExpressionPtr const& { return (*this)[key->String()]; } auto Expression::operator[](ExpressionPtr const& key) && -> ExpressionPtr { return std::move(*this)[key->String()]; } auto Expression::operator[](std::size_t pos) const& -> ExpressionPtr const& { if (pos < List().size()) { return List()[pos]; } throw ExpressionTypeError{ fmt::format("List pos '{}' is out of bounds.", pos)}; } auto Expression::operator[](std::size_t pos) && -> ExpressionPtr { auto&& list = std::move(*this).List(); if (pos < list.size()) { return list[pos]; } throw ExpressionTypeError{ fmt::format("List pos '{}' is out of bounds.", pos)}; } auto Expression::ToJson(Expression::JsonMode mode) const -> nlohmann::json { if (IsBool()) { return Bool(); } if (IsNumber()) { return Number(); } if (IsString()) { return String(); } if (IsArtifact() and mode != JsonMode::NullForNonJson) { return Artifact().ToJson(); } if (IsResult() and mode != JsonMode::NullForNonJson) { auto const& result = Result(); return Expression{map_t{{{"artifact_stage", result.artifact_stage}, {"runfiles", result.runfiles}, {"provides", result.provides}}}} .ToJson(JsonMode::SerializeAllButNodes); } if (IsNode() and mode != JsonMode::NullForNonJson) { switch (mode) { case JsonMode::SerializeAll: return Node().ToJson(); case JsonMode::SerializeAllButNodes: return {{"type", "NODE"}, {"id", ToIdentifier()}}; default: break; } } if (IsList()) { auto json = nlohmann::json::array(); auto const& list = List(); std::transform(list.begin(), list.end(), std::back_inserter(json), [mode](auto const& e) { return e->ToJson(mode); }); return json; } if (IsMap()) { auto json = nlohmann::json::object(); auto const& map = Value()->get(); std::for_each(map.begin(), map.end(), [&](auto const& p) { json.emplace(p.first, p.second->ToJson(mode)); }); return json; } if (IsName() and mode != JsonMode::NullForNonJson) { return Name().ToJson(); } return nlohmann::json{}; } auto Expression::ComputeIsCacheable() const -> bool { // Must be updated whenever we add a new non-cacheable value if (IsName()) { return false; } if (IsResult()) { return Result().is_cacheable; } if (IsNode()) { return Node().IsCacheable(); } if (IsList()) { for (auto const& entry : List()) { if (not entry->IsCacheable()) { return false; } } } if (IsMap()) { for (auto const& [key, entry] : Map()) { if (not entry->IsCacheable()) { return false; } } } return true; } auto Expression::ToString() const -> std::string { return ToJson().dump(); } [[nodiscard]] auto Expression::ToAbbrevString(std::size_t len) const -> std::string { return AbbreviateJson(ToJson(), len); } auto Expression::ToHash() const noexcept -> std::string { return hash_.SetOnceAndGet([this] { return ComputeHash(); }); } auto Expression::IsCacheable() const -> bool { return is_cachable_.SetOnceAndGet([this] { return ComputeIsCacheable(); }); } auto Expression::FromJson(nlohmann::json const& json) noexcept -> ExpressionPtr { if (json.is_null()) { return ExpressionPtr{none_t{}}; } try { // try-catch because json.get<>() could throw, although checked if (json.is_boolean()) { return ExpressionPtr{json.get()}; } if (json.is_number()) { return ExpressionPtr{json.get()}; } if (json.is_string()) { return ExpressionPtr{std::string{json.get()}}; } if (json.is_array()) { auto l = Expression::list_t{}; l.reserve(json.size()); std::transform(json.begin(), json.end(), std::back_inserter(l), [](auto const& j) { return FromJson(j); }); return ExpressionPtr{l}; } if (json.is_object()) { auto m = Expression::map_t::underlying_map_t{}; for (auto const& el : json.items()) { m.emplace(el.key(), FromJson(el.value())); } return ExpressionPtr{Expression::map_t{m}}; } } catch (...) { EnsuresAudit(false); // ensure that the try-block never throws } return ExpressionPtr{nullptr}; } template auto Expression::TypeStringForIndex() const noexcept -> std::string { using var_t = decltype(data_); if (kIndex == data_.index()) { return TypeToString>(); } constexpr auto kSize = std::variant_size_v; if constexpr (kIndex < kSize - 1) { return TypeStringForIndex(); } return TypeToString>(); } auto Expression::TypeString() const noexcept -> std::string { return TypeStringForIndex(); } auto Expression::ComputeHash() const noexcept -> std::string { auto hash = std::string{}; // The type of HashFunction is irrelevant here. It is used for // identification and quick comparison of expressions. SHA256 is used. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; if (IsNone() or IsBool() or IsNumber() or IsString() or IsArtifact() or IsResult() or IsNode() or IsName()) { // just hash the JSON representation, but prepend "@" for artifact, // "=" for result, "#" for node, and "$" for name. std::string prefix; if (IsArtifact()) { prefix = "@"; } else if (IsResult()) { prefix = "="; } else if (IsNode()) { prefix = "#"; } else if (IsName()) { prefix = "$"; } hash = hash_function.PlainHashData(prefix + ToString()).Bytes(); } else { auto hasher = hash_function.MakeHasher(); if (IsList()) { auto list = Value(); hasher.Update("["); for (auto const& el : list->get()) { hasher.Update(el->ToHash()); } } else if (IsMap()) { auto map = Value(); hasher.Update("{"); for (auto const& el : map->get()) { hasher.Update(hash_function.PlainHashData(el.first).Bytes()); hasher.Update(el.second->ToHash()); } } hash = std::move(hasher).Finalize().Bytes(); } return hash; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/expression.hpp000066400000000000000000000324641516554100600321470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP #include #include #include #include #include #include #include #include #include #include // std::move #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/build_engine/expression/target_node.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/multithreading/atomic_value.hpp" #include "src/utils/cpp/hex_string.hpp" class Expression { friend auto operator+(Expression const& /*lhs*/, Expression const& /*rhs*/) -> Expression; public: using none_t = std::monostate; using number_t = double; using artifact_t = ArtifactDescription; using result_t = TargetResult; using node_t = TargetNode; using list_t = std::vector; using map_t = LinkedMap; using name_t = BuildMaps::Base::EntityName; template static consteval auto IsValidType() -> bool { if constexpr (kIndex < std::variant_size_v) { return std::is_same_v< T, std::variant_alternative_t> or IsValidType(); } return false; } class ExpressionTypeError : public std::exception { public: explicit ExpressionTypeError(std::string const& msg) noexcept : msg_{"ExpressionTypeError: " + msg} {} [[nodiscard]] auto what() const noexcept -> char const* final { return msg_.c_str(); } private: std::string msg_; }; Expression() noexcept = default; ~Expression() noexcept = default; Expression(Expression const& other) noexcept = delete; Expression(Expression&& other) noexcept = default; auto operator=(Expression const& other) noexcept = delete; auto operator=(Expression&& other) noexcept = delete; template requires(IsValidType>()) // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) explicit Expression(T&& data) noexcept : data_{std::forward(data)} {} [[nodiscard]] auto IsNone() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsBool() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsNumber() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsString() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsName() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsArtifact() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsResult() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsNode() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsList() const noexcept -> bool { return IsA(); } [[nodiscard]] auto IsMap() const noexcept -> bool { return IsA(); } [[nodiscard]] auto Bool() const -> bool { return Cast(); } [[nodiscard]] auto Number() const -> number_t { return Cast(); } [[nodiscard]] auto Name() const -> name_t { return Cast(); } [[nodiscard]] auto String() const& -> std::string const& { return Cast(); } [[nodiscard]] auto String() && -> std::string { return std::move(*this).Cast(); } [[nodiscard]] auto Artifact() const& -> artifact_t const& { return Cast(); } [[nodiscard]] auto Artifact() && -> artifact_t { return std::move(*this).Cast(); } [[nodiscard]] auto Result() const& -> result_t const& { return Cast(); } [[nodiscard]] auto Result() && -> result_t { return std::move(*this).Cast(); } [[nodiscard]] auto Node() const& -> node_t const& { return Cast(); } [[nodiscard]] auto Node() && -> node_t { return std::move(*this).Cast(); } [[nodiscard]] auto List() const& -> list_t const& { return Cast(); } [[nodiscard]] auto List() && -> list_t { return std::move(*this).Cast(); } [[nodiscard]] auto Map() const& -> map_t const& { return Cast(); } [[nodiscard]] auto Map() && -> map_t { return std::move(*this).Cast(); } [[nodiscard]] auto At(std::string const& key) const& -> std::optional> { auto value = Map().Find(key); if (value) { return std::reference_wrapper{**value}; } return std::nullopt; } [[nodiscard]] auto At( std::string const& key) && -> std::optional { return std::move(*this).Map().Find(key); } template requires(IsValidType>() or std::is_same_v, ExpressionPtr>) [[nodiscard]] auto Get(std::string const& key, T&& default_value) const -> ExpressionPtr { auto value = At(key); if (value) { return value->get(); } if constexpr (std::is_same_v, ExpressionPtr>) { return std::forward(default_value); } else { return ExpressionPtr{std::forward(default_value)}; } } template requires(IsValidType()) [[nodiscard]] auto Value() const& noexcept -> std::optional> { if (GetIndexOf() == data_.index()) { return std::make_optional(std::ref(std::get(data_))); } return std::nullopt; } template requires(IsValidType()) [[nodiscard]] auto Value() && noexcept -> std::optional { if (GetIndexOf() == data_.index()) { return std::make_optional(std::move(std::get(data_))); } return std::nullopt; } template [[nodiscard]] auto operator==(T const& other) const noexcept -> bool { if constexpr (std::is_same_v) { return (&data_ == &other.data_) or (ToHash() == other.ToHash()); } else { return IsValidType() and (GetIndexOf() == data_.index()) and ((static_cast(&data_) == static_cast(&other)) or (std::get(data_) == other)); } } template [[nodiscard]] auto operator!=(T const& other) const noexcept -> bool { return not(*this == other); } [[nodiscard]] auto operator[]( std::string const& key) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](std::string const& key) && -> ExpressionPtr; [[nodiscard]] auto operator[]( ExpressionPtr const& key) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](ExpressionPtr const& key) && -> ExpressionPtr; [[nodiscard]] auto operator[]( std::size_t pos) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](std::size_t pos) && -> ExpressionPtr; enum class JsonMode : std::uint8_t { SerializeAll, SerializeAllButNodes, NullForNonJson }; [[nodiscard]] auto ToJson(JsonMode mode = JsonMode::SerializeAll) const -> nlohmann::json; [[nodiscard]] auto IsCacheable() const -> bool; [[nodiscard]] auto ToString() const -> std::string; [[nodiscard]] auto ToAbbrevString(std::size_t len) const -> std::string; [[nodiscard]] auto ToHash() const noexcept -> std::string; [[nodiscard]] auto ToIdentifier() const noexcept -> std::string { return ToHexString(ToHash()); } [[nodiscard]] static auto FromJson(nlohmann::json const& json) noexcept -> ExpressionPtr; inline static ExpressionPtr const kNone = Expression::FromJson("null"_json); inline static ExpressionPtr const kEmptyMap = Expression::FromJson("{}"_json); inline static ExpressionPtr const kEmptyList = Expression::FromJson("[]"_json); inline static ExpressionPtr const kEmptyMapExpr = Expression::FromJson(R"({"type": "empty_map"})"_json); inline static ExpressionPtr const kEmptyString = Expression::FromJson(R"("")"_json); inline static ExpressionPtr const kOne = Expression::FromJson("1.0"_json); inline static ExpressionPtr const kTrue = Expression::FromJson("true"_json); inline static ExpressionPtr const kFalse = Expression::FromJson("false"_json); private: std::variant data_{none_t{}}; AtomicValue hash_; AtomicValue is_cachable_; template requires(IsValidType()) [[nodiscard]] static consteval auto GetIndexOf() -> std::size_t { static_assert(kIndex < std::variant_size_v, "kIndex out of range"); if constexpr (std::is_same_v< T, std::variant_alternative_t>) { return kIndex; } else { return GetIndexOf(); } } template [[nodiscard]] auto IsA() const noexcept -> bool { return std::holds_alternative(data_); } template [[nodiscard]] auto Cast() const& -> T const& { if (GetIndexOf() == data_.index()) { return std::get(data_); } // throw descriptive ExpressionTypeError throw ExpressionTypeError{ fmt::format("Expression is not of type '{}' but '{}'.", TypeToString(), TypeString())}; } template [[nodiscard]] auto Cast() && -> T { if (GetIndexOf() == data_.index()) { return std::move(std::get(data_)); } // throw descriptive ExpressionTypeError throw ExpressionTypeError{ fmt::format("Expression is not of type '{}' but '{}'.", TypeToString(), TypeString())}; } template requires(Expression::IsValidType()) [[nodiscard]] static auto TypeToString() noexcept -> std::string { if constexpr (std::is_same_v) { return "bool"; } else if constexpr (std::is_same_v) { return "number"; } else if constexpr (std::is_same_v) { return "name"; } else if constexpr (std::is_same_v) { return "string"; } else if constexpr (std::is_same_v) { return "artifact"; } else if constexpr (std::is_same_v) { return "result"; } else if constexpr (std::is_same_v) { return "node"; } else if constexpr (std::is_same_v) { return "list"; } else if constexpr (std::is_same_v) { return "map"; } return "none"; } template [[nodiscard]] auto TypeStringForIndex() const noexcept -> std::string; [[nodiscard]] auto TypeString() const noexcept -> std::string; [[nodiscard]] auto ComputeHash() const noexcept -> std::string; [[nodiscard]] auto ComputeIsCacheable() const -> bool; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(Expression const& e) const noexcept -> std::size_t { auto hash = std::size_t{}; auto bytes = e.ToHash(); std::memcpy(&hash, bytes.data(), std::min(sizeof(hash), bytes.size())); return hash; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/expression_ptr.cpp000066400000000000000000000062621516554100600330240ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include #include #include // std::move #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" ExpressionPtr::ExpressionPtr() noexcept : ptr_{Expression::kNone.ptr_} {} auto ExpressionPtr::operator[]( std::string const& key) const& -> ExpressionPtr const& { return (*ptr_)[key]; } auto ExpressionPtr::operator[](std::string const& key) && -> ExpressionPtr { return (*ptr_)[key]; } auto ExpressionPtr::operator[]( ExpressionPtr const& key) const& -> ExpressionPtr const& { return (*ptr_)[key]; } auto ExpressionPtr::operator[](ExpressionPtr const& key) && -> ExpressionPtr { return (*ptr_)[key]; } auto ExpressionPtr::operator[](std::size_t pos) const& -> ExpressionPtr const& { return (*ptr_)[pos]; } auto ExpressionPtr::operator[](std::size_t pos) && -> ExpressionPtr { return (*ptr_)[pos]; } auto ExpressionPtr::operator<(ExpressionPtr const& other) const -> bool { return ptr_->ToHash() < other.ptr_->ToHash(); } auto ExpressionPtr::operator==(ExpressionPtr const& other) const -> bool { return ptr_ == other.ptr_ or (ptr_ and other.ptr_ and *ptr_ == *other.ptr_); } auto ExpressionPtr::Evaluate( Configuration const& env, FunctionMapPtr const& functions, std::function const& logger, std::function const& annotate_object, std::function const& note_user_context ) const noexcept -> ExpressionPtr { return Evaluator::EvaluateExpression( *this, env, functions, logger, annotate_object, note_user_context); } auto ExpressionPtr::IsCacheable() const noexcept -> bool { return ptr_ and ptr_->IsCacheable(); } auto ExpressionPtr::ToIdentifier() const noexcept -> std::string { return ptr_ ? ptr_->ToIdentifier() : std::string{}; } auto ExpressionPtr::ToJson() const -> nlohmann::json { return ptr_ ? ptr_->ToJson() : nlohmann::json::object(); } auto ExpressionPtr::IsNotNull() const noexcept -> bool { // ExpressionPtr is nullptr in error case and none_t default empty case. return static_cast(ptr_) and not(ptr_->IsNone()); } auto ExpressionPtr::Map() const& -> ExpressionPtr::linked_map_t const& { return ptr_->Map(); } auto ExpressionPtr::Make(linked_map_t&& map) -> ExpressionPtr { return ExpressionPtr{std::move(map)}; } auto std::hash::operator()(ExpressionPtr const& p) const noexcept -> std::size_t { return std::hash{}(*p); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/expression_ptr.hpp000066400000000000000000000111511516554100600330220ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" class Expression; class ExpressionPtr { public: // Initialize to nullptr explicit ExpressionPtr(std::nullptr_t /*ptr*/) noexcept : ptr_{nullptr} {} // Initialize from Expression's variant type or Expression template requires(not std::is_same_v, ExpressionPtr>) // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) explicit ExpressionPtr(T&& data) noexcept : ptr_{std::make_shared(std::forward(data))} {} ExpressionPtr() noexcept; ExpressionPtr(ExpressionPtr const&) noexcept = default; ExpressionPtr(ExpressionPtr&&) noexcept = default; ~ExpressionPtr() noexcept = default; auto operator=(ExpressionPtr const&) noexcept -> ExpressionPtr& = default; auto operator=(ExpressionPtr&&) noexcept -> ExpressionPtr& = default; explicit operator bool() const { return static_cast(ptr_); } [[nodiscard]] auto operator*() & -> Expression& { return *ptr_; } [[nodiscard]] auto operator*() const& -> Expression const& { return *ptr_; } [[nodiscard]] auto operator*() && -> Expression = delete; [[nodiscard]] auto operator->() const& -> Expression const* { return ptr_.get(); } [[nodiscard]] auto operator->() && -> Expression const* = delete; [[nodiscard]] auto operator[]( std::string const& key) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](std::string const& key) && -> ExpressionPtr; [[nodiscard]] auto operator[]( ExpressionPtr const& key) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](ExpressionPtr const& key) && -> ExpressionPtr; [[nodiscard]] auto operator[]( std::size_t pos) const& -> ExpressionPtr const&; [[nodiscard]] auto operator[](std::size_t pos) && -> ExpressionPtr; [[nodiscard]] auto operator<(ExpressionPtr const& other) const -> bool; [[nodiscard]] auto operator==(ExpressionPtr const& other) const -> bool; template [[nodiscard]] auto operator==(T const& other) const -> bool { return ptr_ and *ptr_ == other; } template [[nodiscard]] auto operator!=(T const& other) const -> bool { return not(*this == other); } [[nodiscard]] auto Evaluate( Configuration const& env, FunctionMapPtr const& functions, std::function const& logger = [](std::string const& error) noexcept -> void { Logger::Log(LogLevel::Error, error); }, std::function const& annotate_object = [](auto const& /*unused*/) { return std::string{}; }, std::function const& note_user_context = []() noexcept -> void {}) const noexcept -> ExpressionPtr; [[nodiscard]] auto IsCacheable() const noexcept -> bool; [[nodiscard]] auto ToIdentifier() const noexcept -> std::string; [[nodiscard]] auto ToJson() const -> nlohmann::json; using linked_map_t = LinkedMap; [[nodiscard]] auto IsNotNull() const noexcept -> bool; [[nodiscard]] auto Map() const& -> linked_map_t const&; [[nodiscard]] static auto Make(linked_map_t&& map) -> ExpressionPtr; private: std::shared_ptr ptr_; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(ExpressionPtr const& p) const noexcept -> std::size_t; }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/function_map.hpp000066400000000000000000000025411516554100600324230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP #include #include #include "src/buildtool/build_engine/expression/linked_map.hpp" class ExpressionPtr; class Configuration; using SubExprEvaluator = std::function; using FunctionMap = LinkedMap>; using FunctionMapPtr = FunctionMap::Ptr; #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/linked_map.hpp000066400000000000000000000324201516554100600320430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/multithreading/atomic_value.hpp" #include "src/utils/cpp/hash_combine.hpp" template class LinkedMap; // Default NextPtr for LinkedMap, based on std::shared_ptr. template class LinkedMapPtr { using ptr_t = LinkedMapPtr; using map_t = LinkedMap; public: LinkedMapPtr() noexcept = default; explicit LinkedMapPtr(std::shared_ptr ptr) noexcept : ptr_{std::move(ptr)} {} explicit operator bool() const { return static_cast(ptr_); } [[nodiscard]] auto operator*() const& -> map_t const& { return *ptr_; } [[nodiscard]] auto operator->() const& -> map_t const* { return ptr_.get(); } [[nodiscard]] auto IsNotNull() const noexcept -> bool { return static_cast(ptr_); } [[nodiscard]] auto Map() const& -> map_t const& { return *ptr_; } [[nodiscard]] static auto Make(map_t&& map) -> ptr_t { return ptr_t{std::make_shared(std::move(map))}; } private: std::shared_ptr ptr_{}; }; /// \brief Immutable LinkedMap. /// Uses smart pointers to build up a list of pointer-linked maps. The NextPtr /// that is used internally can be overloaded by any class implementing the /// following methods: /// 1. auto IsNotNull() const noexcept -> bool; /// 2. auto LinkedMap() const& -> LinkedMap const&; /// 3. static auto Make(LinkedMap&&) -> NextPtr; template > class LinkedMap { using item_t = std::pair; using items_t = std::vector; using keys_t = std::vector; using values_t = std::vector; public: using Ptr = NextPtr; // When merging maps, we always rely on entries being traversed in key // order; so keep the underlying map an ordered data structure. using underlying_map_t = std::map; static constexpr auto MakePtr(underlying_map_t map) -> Ptr { return Ptr::Make(LinkedMap{std::move(map)}); } static constexpr auto MakePtr(item_t item) -> Ptr { return Ptr::Make(LinkedMap{std::move(item)}); } static constexpr auto MakePtr(K key, V value) -> Ptr { return Ptr::Make( LinkedMap{std::move(key), std::move(value)}); } static constexpr auto MakePtr(Ptr next, Ptr content) -> Ptr { return Ptr::Make(LinkedMap{next, content}); } static constexpr auto MakePtr(Ptr next, underlying_map_t map) -> Ptr { return Ptr::Make(LinkedMap{next, std::move(map)}); } static constexpr auto MakePtr(Ptr const& next, item_t item) -> Ptr { return Ptr::Make(LinkedMap{next, std::move(item)}); } static constexpr auto MakePtr(Ptr const& next, K key, V value) -> Ptr { return Ptr::Make( LinkedMap{next, std::move(key), std::move(value)}); } explicit LinkedMap(underlying_map_t map) noexcept : map_{std::move(map)} {} explicit LinkedMap(item_t item) noexcept { map_.emplace(std::move(item)); } LinkedMap(K key, V val) noexcept { map_.emplace(std::move(key), std::move(val)); } LinkedMap(Ptr next, Ptr content) noexcept : next_{std::move(next)}, content_{std::move(content)} {} LinkedMap(Ptr next, underlying_map_t map) noexcept : next_{std::move(next)}, map_{std::move(map)} {} LinkedMap(Ptr next, item_t item) noexcept : next_{std::move(next)} { map_.emplace(std::move(item)); } LinkedMap(Ptr next, K key, V val) noexcept : next_{std::move(next)} { map_.emplace(std::move(key), std::move(val)); } LinkedMap() noexcept = default; LinkedMap(LinkedMap const& other) noexcept = delete; LinkedMap(LinkedMap&& other) noexcept = default; ~LinkedMap() noexcept = default; auto operator=(LinkedMap const& other) noexcept = delete; auto operator=(LinkedMap&& other) noexcept = delete; [[nodiscard]] auto contains(K const& key) const noexcept -> bool { return static_cast(Find(key)); } [[nodiscard]] auto at(K const& key) const& -> V const& { auto value = Find(key); if (value) { return **value; } throw std::out_of_range{fmt::format("Missing key {}", key)}; } [[nodiscard]] auto at(K const& key) && -> V { auto value = Find(key); if (value) { return std::move(*value); } throw std::out_of_range{fmt::format("Missing key {}", key)}; } [[nodiscard]] auto operator[](K const& key) const& -> V const& { return at(key); } [[nodiscard]] auto empty() const noexcept -> bool { return (content_.IsNotNull() ? content_.Map().empty() : map_.empty()) and (not next_.IsNotNull() or next_.Map().empty()); } [[nodiscard]] auto Find(K const& key) const& noexcept -> std::optional { if (content_.IsNotNull()) { auto val = content_.Map().Find(key); if (val) { return val; } } else { auto it = map_.find(key); if (it != map_.end()) { return &it->second; } } if (next_.IsNotNull()) { auto val = next_.Map().Find(key); if (val) { return val; } } return std::nullopt; } [[nodiscard]] auto Find(K const& key) && noexcept -> std::optional { if (content_.IsNotNull()) { auto val = content_.Map().Find(key); if (val) { return **val; } } else { auto it = map_.find(key); if (it != map_.end()) { return std::move(it->second); } } if (next_.IsNotNull()) { auto val = next_.Map().Find(key); if (val) { return **val; } } return std::nullopt; } [[nodiscard]] auto FindConflictingDuplicate(LinkedMap const& other) const& noexcept -> std::optional> { auto const& my_items = Items(); auto const& other_items = other.Items(); // Search for duplicates, using that iteration over the items is // ordered by keys. auto me = my_items.begin(); auto they = other_items.begin(); while (me != my_items.end() and they != other_items.end()) { if (me->first == they->first) { if (not(me->second == they->second)) { return me->first; } ++me; ++they; } else if (me->first < they->first) { ++me; } else { ++they; } } return std::nullopt; } [[nodiscard]] auto FindConflictingDuplicate( LinkedMap const& other) && noexcept = delete; // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto size() const noexcept -> std::size_t { return Items().size(); } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto begin() const& -> typename items_t::const_iterator { return Items().cbegin(); } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto end() const& -> typename items_t::const_iterator { return Items().cend(); } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto cbegin() const& -> typename items_t::const_iterator { return begin(); } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto cend() const& -> typename items_t::const_iterator { return end(); } [[nodiscard]] auto begin() && -> typename items_t::const_iterator = delete; [[nodiscard]] auto end() && -> typename items_t::const_iterator = delete; [[nodiscard]] auto cbegin() && -> typename items_t::const_iterator = delete; [[nodiscard]] auto cend() && -> typename items_t::const_iterator = delete; // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto operator==( LinkedMap const& other) const noexcept -> bool { return this == &other or (this->empty() and other.empty()) or this->Items() == other.Items(); } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto Items() const& -> items_t const& { return items_.SetOnceAndGet([this] { return ComputeSortedItems(); }); } [[nodiscard]] auto Items() && = delete; // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto Keys() const -> keys_t { auto keys = keys_t{}; auto const& items = Items(); keys.reserve(items.size()); std::transform(items.begin(), items.end(), std::back_inserter(keys), [](auto const& item) { return item.first; }); return keys; } // NOTE: Expensive, needs to compute sorted items. [[nodiscard]] auto Values() const -> values_t { auto values = values_t{}; auto const& items = Items(); values.reserve(items.size()); std::transform(items.begin(), items.end(), std::back_inserter(values), [](auto const& item) { return item.second; }); return values; } private: Ptr next_{}; // map that is shadowed by this map Ptr content_{}; // content of this map if set underlying_map_t map_{}; // content of this map if content_ is not set AtomicValue items_{}; [[nodiscard]] auto ComputeSortedItems() const noexcept -> items_t { auto size = content_.IsNotNull() ? content_.Map().size() : map_.size(); if (next_.IsNotNull()) { size += next_.Map().size(); } auto items = items_t{}; items.reserve(size); auto empty = items_t{}; auto map_copy = items_t{}; typename items_t::const_iterator citemsit; typename items_t::const_iterator citemsend; typename items_t::const_iterator nitemsit; typename items_t::const_iterator nitemsend; if (content_.IsNotNull()) { auto const& citems = content_.Map().Items(); citemsit = citems.begin(); citemsend = citems.end(); } else { map_copy.reserve(map_.size()); map_copy.insert(map_copy.end(), map_.begin(), map_.end()); citemsit = map_copy.begin(); citemsend = map_copy.end(); } if (next_.IsNotNull()) { auto const& nitems = next_.Map().Items(); nitemsit = nitems.begin(); nitemsend = nitems.end(); } else { nitemsit = empty.begin(); nitemsend = empty.end(); } while (citemsit != citemsend and nitemsit != nitemsend) { if (citemsit->first == nitemsit->first) { items.push_back(*citemsit); ++citemsit; ++nitemsit; } else if (citemsit->first < nitemsit->first) { items.push_back(*citemsit); ++citemsit; } else { items.push_back(*nitemsit); ++nitemsit; } } // No more comparisons to be made; copy over the remaining // entries items.insert(items.end(), citemsit, citemsend); items.insert(items.end(), nitemsit, nitemsend); return items; } }; namespace std { template struct hash> { [[nodiscard]] auto operator()(LinkedMap const& m) const noexcept -> std::size_t { size_t seed{}; for (auto const& e : m) { hash_combine(&seed, e.first); hash_combine(&seed, e.second); } return seed; } }; template struct hash> { [[nodiscard]] auto operator()(LinkedMapPtr const& p) const noexcept -> std::size_t { return std::hash>{}(*p); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/target_node.cpp000066400000000000000000000024661516554100600322350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/target_node.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" auto TargetNode::Abstract::IsCacheable() const noexcept -> bool { return target_fields->IsCacheable(); } auto TargetNode::ToJson() const -> nlohmann::json { if (IsValue()) { return {{"type", "VALUE_NODE"}, {"result", GetValue()->ToJson()}}; } auto const& data = GetAbstract(); return {{"type", "ABSTRACT_NODE"}, {"node_type", data.node_type}, {"string_fields", data.string_fields->ToJson()}, {"target_fields", data.target_fields->ToJson( Expression::JsonMode::SerializeAllButNodes)}}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/target_node.hpp000066400000000000000000000062621516554100600322400ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP #define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP #include #include // IWYU pragma: keep #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" class TargetNode { using Value = ExpressionPtr; // store result type public: struct Abstract { std::string node_type; // arbitrary string that maps to rule ExpressionPtr string_fields; // map to list of strings ExpressionPtr target_fields; // map to list of targets [[nodiscard]] auto IsCacheable() const noexcept -> bool; }; template requires(std::is_same_v or std::is_same_v) explicit TargetNode(NodeType node) : data_{std::move(node)}, is_cacheable_{std::get(data_).IsCacheable()} {} [[nodiscard]] auto IsCacheable() const noexcept -> bool { return is_cacheable_; } [[nodiscard]] auto IsValue() const noexcept { return std::holds_alternative(data_); } [[nodiscard]] auto IsAbstract() const noexcept { return std::holds_alternative(data_); } [[nodiscard]] auto GetValue() const -> Value const& { return std::get(data_); } [[nodiscard]] auto GetAbstract() const -> Abstract const& { return std::get(data_); } [[nodiscard]] auto operator==(TargetNode const& other) const noexcept -> bool { if (data_.index() != other.data_.index()) { return false; } try { if (IsValue()) { return GetValue() == other.GetValue(); } auto const& abs_l = GetAbstract(); auto const& abs_r = other.GetAbstract(); return abs_l.node_type == abs_r.node_type and abs_l.string_fields == abs_r.string_fields and abs_l.target_fields == abs_r.string_fields; } catch (...) { // should never happen return false; } } [[nodiscard]] auto ToString() const noexcept -> std::string { try { return ToJson().dump(); } catch (...) { // should never happen return {}; } } [[nodiscard]] auto ToJson() const -> nlohmann::json; private: std::variant data_; bool is_cacheable_; }; #endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/target_result.cpp000066400000000000000000000510141516554100600326170ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/target_result.hpp" #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/target_node.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { // Serialize artifact description to JSON. If replacements is set, replace // non-known artifacts by known artifacts from replacement. Throws // bad_variant_access if expr is not an artifact or runtime_error if no // replacement is found. [[nodiscard]] auto SerializeArtifactDescription( ExpressionPtr const& expr, std::unordered_map const& replacements) -> nlohmann::json { if (not replacements.empty()) { auto const& artifact = expr->Artifact(); if (not expr->Artifact().IsKnown()) { if (auto it = replacements.find(artifact); it != replacements.end()) { auto const& info = it->second; return ArtifactDescription::CreateKnown(info.digest, info.type) .ToJson(); } throw std::runtime_error{ "No replacement for non-known artifact found."}; } } return expr->ToJson(); } // forward declare as we have mutually recursive functions auto SerializeTargetResultWithReplacement( TargetResult const& result, std::unordered_map const& replacements = {}) -> nlohmann::json; // Serialize arbitrary expression to JSON. This and any sub-expressions will be // collected in `nodes`. Any possible duplicate will be collected only once. As // pure JSON values can coincide with our JSON encoding of artifacts, the hash // of artifact expressions is recorded in `provided_artifacts` to differentiate // them from non-artifacts. Similarly for result and node values. // If replacements is set, replace any contained // non-known artifact by known artifact from replacement. Throws runtime_error // if no replacement is found. [[nodiscard]] auto SerializeExpression( gsl::not_null*> const& nodes, gsl::not_null*> const& provided_artifacts, gsl::not_null*> const& provided_nodes, gsl::not_null*> const& provided_results, ExpressionPtr const& expr, std::unordered_map const& replacements) -> std::string { auto id = ToHexString(expr->ToHash()); if (not nodes->contains(id)) { auto json = nlohmann::json(); if (expr->IsMap()) { auto const& map = expr->Map(); std::unordered_map hashes{}; hashes.reserve(map.size()); for (auto const& [key, val] : map) { auto hash = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, val, replacements); hashes[key] = std::move(hash); } json = std::move(hashes); } else if (expr->IsList()) { auto const& list = expr->List(); std::vector hashes{}; hashes.reserve(list.size()); for (auto const& val : list) { auto hash = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, val, replacements); hashes.emplace_back(std::move(hash)); } json = std::move(hashes); } else if (expr->IsNode()) { auto const& node = expr->Node(); provided_nodes->emplace_back(id); if (node.IsValue()) { auto hash = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, node.GetValue(), replacements); json = nlohmann::json{{"type", "VALUE_NODE"}, {"result", hash}}; } else { auto const& data = node.GetAbstract(); auto string_fields = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, data.string_fields, replacements); auto target_fields = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, data.target_fields, replacements); json = nlohmann::json{{"type", "ABSTRACT_NODE"}, {"node_type", data.node_type}, {"string_fields", string_fields}, {"target_fields", target_fields}}; } } else if (expr->IsResult()) { provided_results->emplace_back(id); auto const& result = expr->Result(); auto artifact_stage = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, result.artifact_stage, replacements); auto runfiles = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, result.runfiles, replacements); auto provides = SerializeExpression(nodes, provided_artifacts, provided_nodes, provided_results, result.provides, replacements); json = nlohmann::json({{"artifact_stage", artifact_stage}, {"runfiles", runfiles}, {"provides", provides}}); } else if (expr->IsArtifact()) { provided_artifacts->emplace_back(id); json = SerializeArtifactDescription(expr, replacements); } else { json = expr->ToJson(); } (*nodes)[id] = std::move(json); } return id; } [[nodiscard]] auto DeserializeExpression( HashFunction::Type hash_type, nlohmann::json const& entry, nlohmann::json const& nodes, std::unordered_set const& provided_artifacts, std::unordered_set const& provided_nodes, std::unordered_set const& provided_results, gsl::not_null*> const& sofar) -> ExpressionPtr { auto id = entry.get(); auto it = sofar->find(id); if (it != sofar->end()) { return it->second; } auto const& json = nodes.at(id); if (json.is_object()) { if (provided_artifacts.contains(id)) { if (auto artifact = ArtifactDescription::FromJson(hash_type, json)) { auto result = ExpressionPtr{*artifact}; sofar->emplace(id, result); return result; } return ExpressionPtr{nullptr}; } if (provided_nodes.contains(id)) { if (json["type"] == "ABSTRACT_NODE") { auto node_type = json["node_type"].get(); auto target_fields = DeserializeExpression(hash_type, json["target_fields"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto string_fields = DeserializeExpression(hash_type, json["string_fields"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto result = ExpressionPtr{TargetNode{ TargetNode::Abstract{.node_type = node_type, .string_fields = string_fields, .target_fields = target_fields}}}; sofar->emplace(id, result); return result; } if (json["type"] == "VALUE_NODE") { auto value = DeserializeExpression(hash_type, json["result"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto result = ExpressionPtr{TargetNode{value}}; sofar->emplace(id, result); return result; } return ExpressionPtr{nullptr}; } if (provided_results.contains(id)) { auto artifact_stage = DeserializeExpression(hash_type, json["artifact_stage"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto runfiles = DeserializeExpression(hash_type, json["runfiles"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto provides = DeserializeExpression(hash_type, json["provides"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); if (artifact_stage and runfiles and provides) { return ExpressionPtr{ TargetResult{.artifact_stage = std::move(artifact_stage), .provides = std::move(provides), .runfiles = std::move(runfiles), .is_cacheable = true}}; } return ExpressionPtr{nullptr}; } Expression::map_t::underlying_map_t map{}; for (auto const& [key, val] : json.items()) { auto new_val = DeserializeExpression(hash_type, val.get(), nodes, provided_artifacts, provided_nodes, provided_results, sofar); if (not new_val) { return new_val; } map.emplace(key, std::move(new_val)); } auto result = ExpressionPtr{Expression::map_t{map}}; sofar->emplace(id, result); return result; } if (json.is_array()) { Expression::list_t list{}; list.reserve(json.size()); for (auto const& val : json) { auto new_val = DeserializeExpression(hash_type, val.get(), nodes, provided_artifacts, provided_nodes, provided_results, sofar); if (not new_val) { return new_val; } list.emplace_back(std::move(new_val)); } auto result = ExpressionPtr{list}; sofar->emplace(id, result); return result; } auto result = Expression::FromJson(json); sofar->emplace(id, result); return result; } // Serialize artifact map to JSON. If replacements is set, replace // non-known artifacts by known artifacts from replacement. Throws runtime_error // if no replacement is found. [[nodiscard]] auto SerializeArtifactMap( ExpressionPtr const& expr, std::unordered_map const& replacements) -> nlohmann::json { if (replacements.empty()) { return expr->ToJson(); } auto const& map = expr->Map(); std::unordered_map artifacts{}; artifacts.reserve(map.size()); for (auto const& [key, val] : map) { artifacts[key] = SerializeArtifactDescription(val, replacements); } return artifacts; } [[nodiscard]] auto DeserializeArtifactMap(HashFunction::Type hash_type, nlohmann::json const& json) -> ExpressionPtr { if (json.is_object()) { Expression::map_t::underlying_map_t map{}; for (auto const& [key, val] : json.items()) { auto artifact = ArtifactDescription::FromJson(hash_type, val); if (not artifact) { return ExpressionPtr{nullptr}; } map.emplace(key, ExpressionPtr{std::move(*artifact)}); } return ExpressionPtr{Expression::map_t{map}}; } return ExpressionPtr{nullptr}; } // Serialize provides map to JSON. If replacements is set, replace // non-known artifacts by known artifacts from replacement. Throws runtime_error // if no replacement is found. [[nodiscard]] auto SerializeProvidesMap( ExpressionPtr const& expr, std::unordered_map const& replacements) -> nlohmann::json { auto provided_artifacts = std::vector{}; auto provided_nodes = std::vector{}; auto provided_results = std::vector{}; auto nodes = std::unordered_map{}; auto entry = SerializeExpression(&nodes, &provided_artifacts, &provided_nodes, &provided_results, expr, replacements); return nlohmann::json{{"entry", std::move(entry)}, {"nodes", std::move(nodes)}, {"provided_artifacts", std::move(provided_artifacts)}, {"provided_nodes", std::move(provided_nodes)}, {"provided_results", std::move(provided_results)} }; } auto JsonSet(nlohmann::json const& j) -> std::unordered_set { std::unordered_set result{}; result.reserve(j.size()); for (auto const& it : j) { result.emplace(it.get()); } return result; } [[nodiscard]] auto DeserializeProvidesMap(HashFunction::Type hash_type, nlohmann::json const& json) -> ExpressionPtr { std::unordered_map sofar{}; return DeserializeExpression(hash_type, json["entry"], json["nodes"], JsonSet(json["provided_artifacts"]), JsonSet(json["provided_nodes"]), JsonSet(json["provided_results"]), &sofar); } // Serialize TargetResult to JSON. If replacements is set, replace non-known // artifacts by known artifacts from replacement. Throws runtime_error if no // replacement is found. [[nodiscard]] auto SerializeTargetResultWithReplacement( TargetResult const& result, std::unordered_map const& replacements) -> nlohmann::json { return nlohmann::json{ {"artifacts", SerializeArtifactMap(result.artifact_stage, replacements)}, {"runfiles", SerializeArtifactMap(result.runfiles, replacements)}, {"provides", SerializeProvidesMap(result.provides, replacements)}}; } } // namespace auto TargetResult::ToJson() const -> nlohmann::json { return SerializeTargetResultWithReplacement(*this /* no replacement */); } auto TargetResult::ReplaceNonKnownAndToJson( std::unordered_map const& replacements) const noexcept -> std::optional { try { return SerializeTargetResultWithReplacement(*this, replacements); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Serializing target result to JSON failed with:\n{}", ex.what()); } return std::nullopt; } auto TargetResult::FromJson(HashFunction::Type hash_type, nlohmann::json const& json) noexcept -> std::optional { try { auto artifacts = DeserializeArtifactMap(hash_type, json["artifacts"]); auto runfiles = DeserializeArtifactMap(hash_type, json["runfiles"]); auto provides = DeserializeProvidesMap(hash_type, json["provides"]); if (artifacts and runfiles and provides) { return TargetResult{artifacts, provides, runfiles}; } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Deserializing target result failed with:\n{}", ex.what()); } return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/expression/target_result.hpp000066400000000000000000000046401516554100600326270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP #define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/hash_combine.hpp" struct TargetResult { ExpressionPtr artifact_stage; ExpressionPtr provides; ExpressionPtr runfiles; bool is_cacheable{provides.IsCacheable()}; [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, nlohmann::json const& json) noexcept -> std::optional; [[nodiscard]] auto ToJson() const -> nlohmann::json; [[nodiscard]] auto ReplaceNonKnownAndToJson( std::unordered_map const& replacements) const noexcept -> std::optional; [[nodiscard]] auto operator==(TargetResult const& other) const noexcept -> bool { return artifact_stage == other.artifact_stage and provides == other.provides and runfiles == other.runfiles; } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(TargetResult const& r) noexcept -> std::size_t { auto seed = std::hash{}(r.artifact_stage); hash_combine(&seed, std::hash{}(r.provides)); hash_combine(&seed, std::hash{}(r.runfiles)); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/000077500000000000000000000000001516554100600271525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/TARGETS000066400000000000000000000142601516554100600302110ustar00rootroot00000000000000{ "configured_target": { "type": ["@", "rules", "CC", "library"] , "name": ["configured_target"] , "hdrs": ["configured_target.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "json"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } , "result_map": { "type": ["@", "rules", "CC", "library"] , "name": ["result_map"] , "hdrs": ["result_map.hpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/analysed_target", "graph_information"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } , "target_map": { "type": ["@", "rules", "CC", "library"] , "name": ["target_map"] , "hdrs": ["target_map.hpp"] , "srcs": ["utils.cpp", "built_in_rules.cpp", "export.cpp", "target_map.cpp"] , "private-hdrs": ["built_in_rules.hpp", "export.hpp", "utils.hpp"] , "deps": [ "absent_target_map" , "configured_target" , "result_map" , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "directory_map"] , ["src/buildtool/build_engine/base_maps", "rule_map"] , ["src/buildtool/build_engine/base_maps", "source_map"] , ["src/buildtool/build_engine/base_maps", "targets_file_map"] , ["src/buildtool/main", "analyse_context"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/analysed_target", "graph_information"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/base_maps", "expression_function"] , ["src/buildtool/build_engine/base_maps", "field_reader"] , ["src/buildtool/build_engine/base_maps", "module_name"] , ["src/buildtool/build_engine/base_maps", "user_rule"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/expression", "linked_map"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "json"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] , ["src/utils/cpp", "vector"] ] } , "target_map_testable_internals": { "type": ["@", "rules", "CC", "library"] , "tainted": ["test"] , "name": ["target_map_testable_internals"] , "hdrs": ["utils.hpp"] , "deps": [ "configured_target" , "target_map" , ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "tree"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } , "absent_target_map": { "type": ["@", "rules", "CC", "library"] , "name": ["absent_target_map"] , "hdrs": ["absent_target_map.hpp"] , "srcs": ["absent_target_map.cpp"] , "deps": [ "configured_target" , "result_map" , ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/main", "analyse_context"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/analysed_target", "graph_information"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "json"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/absent_target_map.cpp000066400000000000000000000311641516554100600333420ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/json.hpp" #endif // BOOTSTRAP_BUILD_TOOL #ifndef BOOTSTRAP_BUILD_TOOL namespace { void WithFlexibleVariables( const gsl::not_null& context, const BuildMaps::Target::ConfiguredTarget& key, std::vector flexible_vars, const BuildMaps::Target::AbsentTargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::AbsentTargetMap::SetterPtr& setter, const BuildMaps::Target::AbsentTargetMap::LoggerPtr& logger, const gsl::not_null result_map, BuildMaps::Target::ServeFailureLogReporter* serve_failure_reporter) { auto effective_config = key.config.Prune(flexible_vars); if (key.config != effective_config) { (*subcaller)( {BuildMaps::Target::ConfiguredTarget{.target = key.target, .config = effective_config}}, [setter](auto const& values) { AnalysedTargetPtr result = *(values[0]); (*setter)(std::move(result)); }, logger); return; } // TODO(asartori): avoid code duplication in export.cpp context->statistics->IncrementExportsFoundCounter(); auto target_name = key.target.GetNamedTarget(); auto repo_key = context->repo_config->RepositoryKey(*context->storage, target_name.repository); if (not repo_key) { (*logger)(fmt::format("Failed to obtain repository key for repo \"{}\"", target_name.repository), /*fatal=*/true); return; } auto target_cache_key = context->storage->TargetCache().ComputeKey( *repo_key, target_name, effective_config); if (not target_cache_key) { (*logger)(fmt::format("Could not produce cache key for target {}", key.target.ToString()), /*fatal=*/true); return; } std::optional> target_cache_value{std::nullopt}; target_cache_value = context->storage->TargetCache().Read(*target_cache_key); bool from_just_serve = false; if (not target_cache_value and context->serve != nullptr) { auto task = fmt::format("[{},{}]", key.target.ToString(), PruneJson(effective_config.ToJson()).dump()); Logger::Log( LogLevel::Debug, "Querying serve endpoint for absent export target {} with key {}", task, key.target.ToString()); context->progress->TaskTracker().Start(task); auto res = context->serve->ServeTarget(*target_cache_key, *repo_key); // process response from serve endpoint if (not res) { // report target not found (*logger)(fmt::format("Absent target {} was not found on serve " "endpoint", key.target.ToString()), /*fatal=*/true); return; } switch (auto const& ind = res->index(); ind) { case 0: { if (serve_failure_reporter != nullptr) { (*serve_failure_reporter)(key, std::get<0>(*res)); } // target found but failed to analyse/build: log it as fatal (*logger)( fmt::format("Failure to remotely analyse or build absent " "target {}\nDetailed log available on the " "remote-execution endpoint as blob {}", key.target.ToString(), std::get<0>(*res)), /*fatal=*/true); return; } case 1: // fallthrough case 2: { // Other errors, including INTERNAL: log it as fatal (*logger)(fmt::format( "While querying serve endpoint for absent export " "target {}:\n{}", key.target.ToString(), ind == 1 ? std::get<1>(*res) : std::get<2>(*res)), /*fatal=*/true); return; } default: { // index == 3 target_cache_value = std::get<3>(*res); context->progress->TaskTracker().Stop(task); from_just_serve = true; } } } if (not target_cache_value) { (*logger)(fmt::format("Could not get target cache value for key {}", target_cache_key->Id().ToString()), /*fatal=*/true); return; } auto const& [entry, info] = *target_cache_value; if (auto result = entry.ToResult()) { auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{ .target = key.target, .config = effective_config}), {}, {}, {}}; auto analysis_result = std::make_shared( *result, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{flexible_vars.begin(), flexible_vars.end()}, std::set{}, entry.ToImplied(), deps_info); analysis_result = result_map->Add( key.target, effective_config, analysis_result, std::nullopt, true); Logger::Log(LogLevel::Performance, "Absent export target {} taken from {}: {} -> {}", key.target.ToString(), (from_just_serve ? "serve endpoint" : "cache"), target_cache_key->Id().ToString(), info.ToString()); (*setter)(std::move(analysis_result)); if (from_just_serve) { context->statistics->IncrementExportsServedCounter(); } else { context->statistics->IncrementExportsCachedCounter(); } return; } (*logger)(fmt::format("Reading target entry for key {} failed", target_cache_key->Id().ToString()), /*fatal=*/true); } } // namespace #endif // BOOTSTRAP_BUILD_TOOL auto BuildMaps::Target::CreateAbsentTargetVariablesMap( const gsl::not_null& context, std::size_t jobs) -> AbsentTargetVariablesMap { #ifdef BOOTSTRAP_BUILD_TOOL auto target_variables = [](auto /*ts*/, auto /*setter*/, auto /*logger*/, auto /*subcaller*/, auto /*key*/) {}; #else auto target_variables = [context](auto /*ts*/, auto setter, auto logger, auto /*subcaller*/, auto key) { std::optional> vars; if (context->serve != nullptr) { vars = context->serve->ServeTargetVariables( key.target_root_id, key.target_file, key.target); } if (not vars) { (*logger)(fmt::format("Failed to obtain flexible config variables " "for absent target {}", key.target), /*fatal=*/true); return; } (*setter)(std::move(vars.value())); }; #endif return AbsentTargetVariablesMap{target_variables, jobs}; } auto BuildMaps::Target::CreateAbsentTargetMap( const gsl::not_null& context, const gsl::not_null& result_map, const gsl::not_null& absent_variables, std::size_t jobs, BuildMaps::Target::ServeFailureLogReporter* serve_failure_reporter) -> AbsentTargetMap { #ifndef BOOTSTRAP_BUILD_TOOL auto target_reader = [context, result_map, absent_variables, serve_failure_reporter]( auto ts, auto setter, auto logger, auto subcaller, auto key) { // assumptions: // - target with absent targets file requested // - ServeApi correctly configured auto const& repo_name = key.target.ToModule().repository; auto target_root_id = context->repo_config->TargetRoot(repo_name)->GetAbsentTreeId(); if (not target_root_id) { (*logger)(fmt::format("Failed to get the target root id for " "repository \"{}\"", repo_name), /*fatal=*/true); return; } std::filesystem::path module{key.target.ToModule().module}; auto vars_request = AbsentTargetDescription{ .target_root_id = *target_root_id, .target_file = (module / *(context->repo_config->TargetFileName( repo_name))) .string(), .target = key.target.GetNamedTarget().name}; absent_variables->ConsumeAfterKeysReady( ts, {vars_request}, [context, key, setter, logger, serve_failure_reporter, result_map, subcaller](auto const& values) { WithFlexibleVariables(context, key, *(values[0]), subcaller, setter, logger, result_map, serve_failure_reporter); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While requested the flexible " "variables of {}:\n{}", target.ToString(), msg), fatal); }); }; #else auto target_reader = [](auto /*ts*/, auto /*setter*/, auto /*logger*/, auto /*subcaller*/, auto /*key*/) {}; #endif return AbsentTargetMap{target_reader, jobs}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/absent_target_map.hpp000066400000000000000000000057461516554100600333560ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/utils/cpp/hash_combine.hpp" namespace BuildMaps::Target { struct AbsentTargetDescription { std::string target_root_id; std::string target_file; std::string target; [[nodiscard]] auto operator==( AbsentTargetDescription const& other) const noexcept -> bool { return target_root_id == other.target_root_id and target_file == other.target_file and target == other.target; } }; using AbsentTargetMap = AsyncMapConsumer; using AbsentTargetVariablesMap = AsyncMapConsumer>; using ServeFailureLogReporter = std::function; auto CreateAbsentTargetVariablesMap( const gsl::not_null& context, std::size_t jobs = 0) -> AbsentTargetVariablesMap; auto CreateAbsentTargetMap(const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, std::size_t jobs = 0, ServeFailureLogReporter* serve_failure_reporter = nullptr) -> AbsentTargetMap; } // namespace BuildMaps::Target namespace std { template <> struct hash { [[nodiscard]] auto operator()( const BuildMaps::Target::AbsentTargetDescription& t) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, t.target_root_id); hash_combine(&seed, t.target_file); hash_combine(&seed, t.target); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/built_in_rules.cpp000066400000000000000000002114531516554100600327030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/built_in_rules.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/build_engine/target_map/export.hpp" #include "src/buildtool/build_engine/target_map/utils.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/vector.hpp" namespace { auto const kGenericRuleFields = std::unordered_set{"arguments_config", "cmds", "cwd", "deps", "env", "execution properties", "sh -c", "tainted", "timeout scaling", "type", "out_dirs", "outs"}; auto const kBlobGenRuleFields = std::unordered_set{"arguments_config", "data", "deps", "name", "tainted", "type"}; auto const kTreeRuleFields = std::unordered_set{"arguments_config", "deps", "name", "tainted", "type"}; auto const kInstallRuleFields = std::unordered_set{"arguments_config", "deps", "dirs", "files", "tainted", "type"}; auto const kConfigureRuleFields = std::unordered_set{"arguments_config", "config", "doc", "tainted", "target", "type"}; void ReportArtifactWithDependencyOrigin( const ExpressionPtr& artifact, const std::unordered_map& deps_by_target, std::stringstream* msg) { *msg << " - " << artifact->ToString() << " from\n"; for (auto const& [name, analysis_result] : deps_by_target) { for (auto const& [path, value] : analysis_result->Artifacts().Map()) { if (value == artifact) { *msg << " - " << name.ToString() << ", artifact at " << nlohmann::json(path).dump() << "\n"; } } for (auto const& [path, value] : analysis_result->RunFiles().Map()) { if (value == artifact) { *msg << " - " << name.ToString() << ", runfile at " << nlohmann::json(path).dump() << "\n"; } } } } void ReportStagingConflict( const std::string& location, const ExpressionPtr& stage_a, const ExpressionPtr& stage_b, const std::unordered_map& deps_by_target, const BuildMaps::Target::TargetMap::LoggerPtr& logger) { std::stringstream msg{}; auto artifact_a = stage_a->Get(location, Expression::kNone); auto artifact_b = stage_b->Get(location, Expression::kNone); msg << "Staging conflict on path " << nlohmann::json(location).dump() << " between\n"; ReportArtifactWithDependencyOrigin(artifact_a, deps_by_target, &msg); ReportArtifactWithDependencyOrigin(artifact_b, deps_by_target, &msg); (*logger)(msg.str(), true); } void BlobGenRuleWithDeps( const gsl::not_null& context, const std::vector& transition_keys, const std::vector& dependency_values, const BuildMaps::Base::FieldReader::Ptr& desc, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map, const ObjectType& blob_type) { // Associate keys and values std::unordered_map deps_by_transition; deps_by_transition.reserve(transition_keys.size()); for (std::size_t i = 0; i < transition_keys.size(); ++i) { deps_by_transition.emplace(transition_keys[i], *dependency_values[i]); } auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); auto vars_set = std::unordered_set{}; vars_set.insert(param_vars->begin(), param_vars->end()); for (auto const& dep : dependency_values) { vars_set.insert((*dep)->Vars().begin(), (*dep)->Vars().end()); } auto effective_conf = key.config.Prune(vars_set); std::vector all_deps{}; all_deps.reserve(dependency_values.size()); for (auto const& dep : dependency_values) { all_deps.emplace_back((*dep)->GraphInformation().Node()); } auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{key.target, effective_conf}), all_deps, {}, {}}; auto string_fields_fcts = FunctionMap::MakePtr(FunctionMap::underlying_map_t{ {"outs", [&deps_by_transition, &key, context]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, context->repo_config, deps_by_transition) ->Artifacts()); }}, {"runfiles", [&deps_by_transition, &key, context]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, context->repo_config, deps_by_transition) ->RunFiles()); }}}); auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, param_config, desc->ReadOptionalExpression("tainted", Expression::kEmptyList), logger); if (not got_tainted) { return; } for (auto const& dep : dependency_values) { if (not std::includes(tainted.begin(), tainted.end(), (*dep)->Tainted().begin(), (*dep)->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } } std::set implied_export{}; for (auto const& dep : dependency_values) { implied_export.insert((*dep)->ImpliedExport().begin(), (*dep)->ImpliedExport().end()); } auto name_exp = desc->ReadOptionalExpression( "name", ExpressionPtr{std::string{"out.txt"}}); if (not name_exp) { return; } auto name_val = name_exp.Evaluate( param_config, string_fields_fcts, [logger](auto const& msg) { (*logger)(fmt::format("While evaluating name:\n{}", msg), true); }); if (not name_val) { return; } if (not name_val->IsString()) { (*logger)(fmt::format("name should evaluate to a string, but got {}", name_val->ToString()), true); return; } auto data_exp = desc->ReadOptionalExpression("data", ExpressionPtr{std::string{""}}); if (not data_exp) { return; } auto data_val = data_exp.Evaluate( param_config, string_fields_fcts, [logger](auto const& msg) { (*logger)(fmt::format("While evaluating data:\n{}", msg), true); }); if (not data_val) { return; } if (not data_val->IsString()) { (*logger)(fmt::format("data should evaluate to a string, but got {}", data_val->ToString()), true); return; } // if symlink target, we only accept non-upwards if (IsSymlinkObject(blob_type) and not PathIsNonUpwards(data_val->String())) { (*logger)(fmt::format("data string {} does not constitute a " "non-upwards symlink target path", data_val->String()), true); return; } auto stage = ExpressionPtr{Expression::map_t{ name_val->String(), ExpressionPtr{ArtifactDescription::CreateKnown( ArtifactDigestFactory::HashDataAs( context->storage->GetHashFunction(), data_val->String()), blob_type)}}}; auto analysis_result = std::make_shared( TargetResult{.artifact_stage = stage, .provides = ExpressionPtr{Expression::map_t{}}, .runfiles = stage}, std::vector{}, std::vector{data_val->String()}, std::vector{}, std::vector{}, std::move(vars_set), std::move(tainted), std::move(implied_export), std::move(deps_info)); analysis_result = result_map->Add(key.target, effective_conf, std::move(analysis_result)); (*setter)(std::move(analysis_result)); } void BlobGenRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map, const ObjectType& blob_type) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, IsSymlinkObject(blob_type) ? "symlink target" : "file-generation target", logger); desc->ExpectFields(kBlobGenRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); // Collect dependencies: deps auto const& empty_list = Expression::kEmptyList; auto deps_exp = desc->ReadOptionalExpression("deps", empty_list); if (not deps_exp) { return; } auto deps_value = deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating deps:\n{}", msg), true); }); if (not deps_value) { return; } if (not deps_value->IsList()) { (*logger)(fmt::format("Expected deps to evaluate to a list of targets, " "but found {}", deps_value->ToString()), true); return; } std::vector dependency_keys; std::vector transition_keys; dependency_keys.reserve(deps_value->List().size()); transition_keys.reserve(deps_value->List().size()); auto empty_transition = Configuration{Expression::kEmptyMap}; for (auto const& dep_name : deps_value->List()) { auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &dep_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}", dep_name->ToString(), parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); transition_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, empty_transition}); } (*subcaller)( dependency_keys, [context, transition_keys = std::move(transition_keys), desc, setter, logger, key, result_map, blob_type](auto const& values) { BlobGenRuleWithDeps(context, transition_keys, values, desc, key, setter, logger, result_map, blob_type); }, logger); } void FileGenRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { BlobGenRule(context, desc_json, key, subcaller, setter, logger, result_map, ObjectType::File); } void SymlinkRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { BlobGenRule(context, desc_json, key, subcaller, setter, logger, result_map, ObjectType::Symlink); } void TreeRuleWithDeps( const std::vector& dependency_values, const std::vector& dependency_keys, const std::string& name, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Base::FieldReader::Ptr& desc, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map, std::optional disjoint_overlay) { auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, param_config, desc->ReadOptionalExpression("tainted", Expression::kEmptyList), logger); if (not got_tainted) { return; } for (auto const& dep : dependency_values) { if (not std::includes(tainted.begin(), tainted.end(), (*dep)->Tainted().begin(), (*dep)->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } } std::set implied_export{}; for (auto const& dep : dependency_values) { implied_export.insert((*dep)->ImpliedExport().begin(), (*dep)->ImpliedExport().end()); } auto vars_set = std::unordered_set{}; vars_set.insert(param_vars->begin(), param_vars->end()); for (auto const& dep : dependency_values) { vars_set.insert((*dep)->Vars().begin(), (*dep)->Vars().end()); } auto effective_conf = key.config.Prune(vars_set); std::vector all_deps{}; all_deps.reserve(dependency_values.size()); for (auto const& dep : dependency_values) { all_deps.emplace_back((*dep)->GraphInformation().Node()); } auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{key.target, effective_conf}), all_deps, {}, {}}; // Compute the resulting stage auto result_stage = Expression::map_t::underlying_map_t{}; std::vector trees{}; std::vector tree_overlays{}; if (disjoint_overlay) { TreeOverlay::to_overlay_t dep_trees{}; for (auto const& dep : dependency_values) { std::unordered_map tree_content; for (auto const& [input_path, artifact] : (*dep)->Artifacts()->Map()) { auto norm_path = ToNormalPath(std::filesystem::path{input_path}); tree_content.emplace(std::move(norm_path), artifact->Artifact()); } auto tree = std::make_shared(std::move(tree_content)); auto tree_id = tree->Id(); trees.emplace_back(std::move(tree)); dep_trees.emplace_back(ArtifactDescription::CreateTree(tree_id)); } auto overlay_tree = std::make_shared(std::move(dep_trees), *disjoint_overlay); auto overlay_tree_id = overlay_tree->Id(); tree_overlays.emplace_back(std::move(overlay_tree)); result_stage.emplace( name, ArtifactDescription::CreateTreeOverlay(overlay_tree_id)); } else { auto stage = ExpressionPtr{Expression::map_t{}}; for (auto const& dep : dependency_values) { auto to_stage = ExpressionPtr{ Expression::map_t{(*dep)->RunFiles(), (*dep)->Artifacts()}}; auto dup = stage->Map().FindConflictingDuplicate(to_stage->Map()); if (dup) { std::unordered_map deps_by_target; deps_by_target.reserve(dependency_keys.size()); for (std::size_t i = 0; i < dependency_keys.size(); ++i) { deps_by_target.emplace(dependency_keys[i].target, *dependency_values[i]); } ReportStagingConflict( dup->get(), stage, to_stage, deps_by_target, logger); return; } stage = ExpressionPtr{Expression::map_t{stage, to_stage}}; } // Result is the associated tree, located at name auto conflict = BuildMaps::Target::Utils::tree_conflict(stage); if (conflict) { (*logger)(fmt::format("TREE conflict on subtree {}", *conflict), true); return; } std::unordered_map tree_content; tree_content.reserve(stage->Map().size()); for (auto const& [input_path, artifact] : stage->Map()) { auto norm_path = ToNormalPath(std::filesystem::path{input_path}); tree_content.emplace(std::move(norm_path), artifact->Artifact()); } auto tree = std::make_shared(std::move(tree_content)); auto tree_id = tree->Id(); trees.emplace_back(std::move(tree)); result_stage.emplace(name, ArtifactDescription::CreateTree(tree_id)); } auto result = ExpressionPtr{Expression::map_t{result_stage}}; auto analysis_result = std::make_shared( TargetResult{.artifact_stage = result, .provides = ExpressionPtr{Expression::map_t{}}, .runfiles = result}, std::vector{}, std::vector{}, std::move(trees), std::move(tree_overlays), std::move(vars_set), std::move(tainted), std::move(implied_export), std::move(deps_info)); analysis_result = result_map->Add(key.target, effective_conf, std::move(analysis_result)); (*setter)(std::move(analysis_result)); } void CommonTreeRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map, std::optional disjoint_overlay) { std::string rule_name; if (disjoint_overlay) { rule_name = *disjoint_overlay ? "disjoint_tree_overlay" : "tree overlay"; } else { rule_name = "tree"; } auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, fmt::format("{} target", rule_name), logger); desc->ExpectFields(kTreeRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); // Collect dependencies: deps auto const& empty_list = Expression::kEmptyList; auto deps_exp = desc->ReadOptionalExpression("deps", empty_list); if (not deps_exp) { return; } auto deps_value = deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating deps:\n{}", msg), true); }); if (not deps_value) { return; } if (not deps_value->IsList()) { (*logger)(fmt::format("Expected deps to evaluate to a list of targets, " "but found {}", deps_value->ToString()), true); return; } std::vector dependency_keys; for (auto const& dep_name : deps_value->List()) { auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &dep_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}", dep_name->ToString(), parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); } auto name_exp = desc->ReadOptionalExpression("name", ExpressionPtr{std::string{""}}); if (not name_exp) { return; } auto name_value = name_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating name:\n{}", msg), true); }); if (not name_value) { return; } if (not name_value->IsString()) { (*logger)( fmt::format("Expected name to evaluate to a string, but got {}", name_value->ToString()), true); return; } (*subcaller)( dependency_keys, [name = name_value->String(), dependency_keys, desc, setter, logger, key, result_map, disjoint_overlay](auto const& values) { TreeRuleWithDeps(values, dependency_keys, name, key, desc, setter, logger, result_map, disjoint_overlay); }, logger); } void TreeRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { CommonTreeRule(context, desc_json, key, subcaller, setter, logger, result_map, std::nullopt); } void DisjointTreeOverlayRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { CommonTreeRule( context, desc_json, key, subcaller, setter, logger, result_map, true); } void TreeOverlayRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { CommonTreeRule( context, desc_json, key, subcaller, setter, logger, result_map, false); } void InstallRuleWithDeps( const std::vector& dependency_keys, const std::vector& dependency_values, const BuildMaps::Base::FieldReader::Ptr& desc, const BuildMaps::Target::ConfiguredTarget& key, const std::vector& deps, const std::unordered_map& files, const std::vector>& dirs, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { // Associate keys and values std::unordered_map deps_by_target; deps_by_target.reserve(dependency_keys.size()); for (std::size_t i = 0; i < dependency_keys.size(); ++i) { deps_by_target.emplace(dependency_keys[i].target, *dependency_values[i]); } // Compute the effective dependency on config variables std::unordered_set effective_vars; auto param_vars = desc->ReadStringList("arguments_config"); effective_vars.insert(param_vars->begin(), param_vars->end()); for (auto const& [target_name, target] : deps_by_target) { effective_vars.insert(target->Vars().begin(), target->Vars().end()); } auto effective_conf = key.config.Prune(effective_vars); std::vector all_deps{}; all_deps.reserve(dependency_values.size()); for (auto const& dep : dependency_values) { all_deps.emplace_back((*dep)->GraphInformation().Node()); } auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{key.target, effective_conf}), all_deps, {}, {}}; // Compute and verify taintedness auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, key.config.Prune(*param_vars), desc->ReadOptionalExpression("tainted", Expression::kEmptyList), logger); if (not got_tainted) { return; } for (auto const& dep : dependency_values) { if (not std::includes(tainted.begin(), tainted.end(), (*dep)->Tainted().begin(), (*dep)->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } } // Compute implied export targets std::set implied_export{}; for (auto const& dep : dependency_values) { implied_export.insert((*dep)->ImpliedExport().begin(), (*dep)->ImpliedExport().end()); } // Stage deps (runfiles only) auto stage = ExpressionPtr{Expression::map_t{}}; for (auto const& dep : deps) { auto to_stage = deps_by_target.at(dep)->RunFiles(); auto dup = stage->Map().FindConflictingDuplicate(to_stage->Map()); if (dup) { ReportStagingConflict( dup->get(), stage, to_stage, deps_by_target, logger); return; } stage = ExpressionPtr{Expression::map_t{stage, to_stage}}; } // stage files (artifacts, but fall back to runfiles) auto files_stage = Expression::map_t::underlying_map_t{}; for (auto const& [path, target] : files) { if (stage->Map().contains(path)) { (*logger)(fmt::format("Staging conflict for path {}", path), true); return; } auto artifacts = deps_by_target[target]->Artifacts(); if (artifacts->Map().empty()) { // If no artifacts are present, fall back to runfiles artifacts = deps_by_target[target]->RunFiles(); } if (artifacts->Map().empty()) { (*logger)(fmt::format( "No artifacts or runfiles for {} to be staged to {}", target.ToString(), path), true); return; } if (artifacts->Map().size() != 1) { (*logger)( fmt::format("Not precisely one entry for {} to be staged to {}", target.ToString(), path), true); return; } files_stage.emplace(path, artifacts->Map().Values()[0]); } stage = ExpressionPtr{Expression::map_t{stage, files_stage}}; // stage dirs (artifacts and runfiles) for (auto const& subdir : dirs) { auto subdir_stage = Expression::map_t::underlying_map_t{}; auto dir_path = std::filesystem::path{subdir.second}; auto target = deps_by_target.at(subdir.first); // within a target, artifacts and runfiles may overlap, but artifacts // take preference for (auto const& [path, artifact] : target->Artifacts()->Map()) { subdir_stage.emplace(ToNormalPath(dir_path / path).string(), artifact); } for (auto const& [path, artifact] : target->RunFiles()->Map()) { subdir_stage.emplace(ToNormalPath(dir_path / path).string(), artifact); } auto to_stage = ExpressionPtr{Expression::map_t{subdir_stage}}; auto dup = stage->Map().FindConflictingDuplicate(to_stage->Map()); if (dup) { ReportStagingConflict( dup->get(), stage, to_stage, deps_by_target, logger); return; } stage = ExpressionPtr{Expression::map_t{stage, to_stage}}; } auto conflict = BuildMaps::Target::Utils::tree_conflict(stage); if (conflict) { (*logger)(fmt::format("TREE conflict on subtree {}", *conflict), true); return; } auto const& empty_map = Expression::kEmptyMap; auto result = std::make_shared( TargetResult{ .artifact_stage = stage, .provides = empty_map, .runfiles = stage}, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::move(effective_vars), std::move(tainted), std::move(implied_export), std::move(deps_info)); result = result_map->Add(key.target, effective_conf, std::move(result)); (*setter)(std::move(result)); } void InstallRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, "install target", logger); desc->ExpectFields(kInstallRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); // Collect dependencies: deps auto const& empty_list = Expression::kEmptyList; auto deps_exp = desc->ReadOptionalExpression("deps", empty_list); if (not deps_exp) { return; } auto deps_value = deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating deps:\n{}", msg), true); }); if (not deps_value) { return; } if (not deps_value->IsList()) { (*logger)(fmt::format("Expected deps to evaluate to a list of targets, " "but found {}", deps_value->ToString()), true); return; } std::vector dependency_keys; std::vector deps; deps.reserve(deps_value->List().size()); for (auto const& dep_name : deps_value->List()) { auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &dep_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}", dep_name->ToString(), parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); deps.emplace_back(*dep_target); } // Collect dependencies: files auto const& empty_map = Expression::kEmptyMap; auto files_exp = desc->ReadOptionalExpression("files", empty_map); if (not files_exp) { return; } if (not files_exp->IsMap()) { (*logger)(fmt::format("Expected files to be a map of target " "expressions, but found {}", files_exp->ToString()), true); return; } auto files = std::unordered_map{}; files.reserve(files_exp->Map().size()); for (auto const& [path, dep_exp] : files_exp->Map()) { auto dep_name = dep_exp.Evaluate( param_config, {}, [&logger, &path = path](auto const& msg) { (*logger)( fmt::format( "While evaluating files entry for {}:\n{}", path, msg), true); }); if (not dep_name) { return; } auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &dep_name, &path = path](std::string const& parse_err) { (*logger)(fmt::format("Parsing file entry {} for key {} failed " "with:\n{}", dep_name->ToString(), path, parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); files.emplace(path, *dep_target); } // Collect dependencies: dirs auto dirs_exp = desc->ReadOptionalExpression("dirs", empty_list); if (not dirs_exp) { return; } auto dirs_value = dirs_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating deps:\n{}", msg), true); }); if (not dirs_value) { return; } if (not dirs_value->IsList()) { (*logger)(fmt::format("Expected dirs to evaluate to a list of " "path-target pairs, but found {}", dirs_value->ToString()), true); return; } auto dirs = std::vector>{}; dirs.reserve(dirs_value->List().size()); for (auto const& entry : dirs_value->List()) { if (not entry->IsList() or entry->List().size() != 2 or not entry->List()[1]->IsString()) { (*logger)(fmt::format("Expected dirs to evaluate to a list of " "target-path pairs, but found entry {}", entry->ToString()), true); return; } auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( entry->List()[0], key.target, context->repo_config, [&logger, &entry](std::string const& parse_err) { (*logger)(fmt::format("Parsing dir entry {} for path {} failed " "with:\n{}", entry->List()[0]->ToString(), entry->List()[1]->String(), parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); dirs.emplace_back(*dep_target, entry->List()[1]->String()); } (*subcaller)( dependency_keys, [dependency_keys, deps = std::move(deps), files = std::move(files), dirs = std::move(dirs), desc, setter, logger, key, result_map](auto const& values) { InstallRuleWithDeps(dependency_keys, values, desc, key, deps, files, dirs, setter, logger, result_map); }, logger); } void GenericRuleWithDeps( const std::vector& transition_keys, const std::vector& dependency_values, const BuildMaps::Base::FieldReader::Ptr& desc, const BuildMaps::Target::ConfiguredTarget& key, const gsl::not_null& repo_config, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { // Associate dependency keys with values std::unordered_map deps_by_transition; deps_by_transition.reserve(transition_keys.size()); for (std::size_t i = 0; i < transition_keys.size(); ++i) { deps_by_transition.emplace(transition_keys[i], *dependency_values[i]); } // Compute the effective dependency on config variables std::unordered_set effective_vars; auto param_vars = desc->ReadStringList("arguments_config"); effective_vars.insert(param_vars->begin(), param_vars->end()); for (auto const& [transition, target] : deps_by_transition) { effective_vars.insert(target->Vars().begin(), target->Vars().end()); } auto effective_conf = key.config.Prune(effective_vars); std::vector all_deps{}; all_deps.reserve(dependency_values.size()); for (auto const& dep : dependency_values) { all_deps.emplace_back((*dep)->GraphInformation().Node()); } auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{key.target, effective_conf}), all_deps, {}, {}}; // Compute and verify taintedness auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, key.config.Prune(*param_vars), desc->ReadOptionalExpression("tainted", Expression::kEmptyList), logger); if (not got_tainted) { return; } for (auto const& dep : dependency_values) { if (not std::includes(tainted.begin(), tainted.end(), (*dep)->Tainted().begin(), (*dep)->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } } // Compute implied export targets std::set implied_export{}; for (auto const& dep : dependency_values) { implied_export.insert((*dep)->ImpliedExport().begin(), (*dep)->ImpliedExport().end()); } // Evaluate cmd, outs, env auto string_fields_fcts = FunctionMap::MakePtr(FunctionMap::underlying_map_t{ {"outs", [&deps_by_transition, &key, repo_config]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, repo_config, deps_by_transition) ->Artifacts()); }}, {"runfiles", [&deps_by_transition, &key, repo_config]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, repo_config, deps_by_transition) ->RunFiles()); }}}); auto const& empty_list = Expression::kEmptyList; auto param_config = key.config.Prune(*param_vars); auto outs_exp = desc->ReadOptionalExpression("outs", empty_list); auto out_dirs_exp = desc->ReadOptionalExpression("out_dirs", empty_list); std::vector outs{}; std::vector out_dirs{}; if (outs_exp) { auto outs_value = outs_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating outs:\n{}", msg), true); }); if (not outs_value) { return; } if (not outs_value->IsList()) { (*logger)(fmt::format("outs has to evaluate to a list of " "strings, but found {}", outs_value->ToString()), true); return; } if (not outs_value->List().empty()) { outs.reserve(outs_value->List().size()); for (auto const& x : outs_value->List()) { if (not x->IsString()) { (*logger)(fmt::format("outs has to evaluate to a list of " "strings, but found entry {}", x->ToString()), true); return; } outs.emplace_back(ToNormalPath(x->String()).string()); } } } if (out_dirs_exp) { auto out_dirs_value = out_dirs_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating out_dirs:\n{}", msg), true); }); if (not out_dirs_value) { return; } if (not out_dirs_value->IsList()) { (*logger)(fmt::format("out_dirs has to evaluate to a list of " "strings, but found {}", out_dirs_value->ToString()), true); return; } if (not out_dirs_value->List().empty()) { out_dirs.reserve(out_dirs_value->List().size()); for (auto const& x : out_dirs_value->List()) { if (not x->IsString()) { (*logger)( fmt::format("out_dirs has to evaluate to a list of " "strings, but found entry {}", x->ToString()), true); return; } out_dirs.emplace_back(ToNormalPath(x->String()).string()); } } } if (outs.empty() and out_dirs.empty()) { (*logger)( R"(At least one of "outs" and "out_dirs" must be specified for "generic")", true); return; } sort_and_deduplicate(&outs); sort_and_deduplicate(&out_dirs); // looking for same paths in both outs and out_dirs std::vector intersection; std::set_intersection(outs.begin(), outs.end(), out_dirs.begin(), out_dirs.end(), std::back_inserter(intersection)); if (not intersection.empty()) { (*logger)(fmt::format("outs and out_dirs for generic must be disjoint. " "Found repeated entries:\n{}", nlohmann::json(intersection).dump()), true); return; } auto cmd_exp = desc->ReadOptionalExpression("cmds", empty_list); if (not cmd_exp) { return; } auto cmd_value = cmd_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating cmds:\n{}", msg), true); }); if (not cmd_value) { return; } if (not cmd_value->IsList()) { (*logger)(fmt::format( "cmds has to evaluate to a list of strings, but found {}", cmd_value->ToString()), true); return; } std::stringstream cmd_ss{}; for (auto const& x : cmd_value->List()) { if (not x->IsString()) { (*logger)(fmt::format("cmds has to evaluate to a list of strings, " "but found entry {}", x->ToString()), true); return; } cmd_ss << x->String(); cmd_ss << "\n"; } auto cwd_exp = desc->ReadOptionalExpression("cwd", Expression::kEmptyString); if (not cwd_exp) { return; } auto cwd_value = cwd_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating cwd:\n{}", msg), true); }); if (not cwd_value) { return; } if (not cwd_value->IsString()) { (*logger)(fmt::format("cwd has to evaluate to a string, but found {}", cwd_value->ToString()), true); return; } if (not PathIsNonUpwards(cwd_value->String())) { (*logger)(fmt::format("cwd has to evaluate to a non-upwards relative " "path, but found {}", cwd_value->ToString()), true); return; } auto const& empty_map_exp = Expression::kEmptyMapExpr; auto env_exp = desc->ReadOptionalExpression("env", empty_map_exp); if (not env_exp) { return; } auto env_val = env_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating env:\n{}", msg), true); }); if (not env_val) { return; } if (not env_val->IsMap()) { (*logger)( fmt::format("env has to evaluate to map of strings, but found {}", env_val->ToString()), true); return; } for (auto const& [var_name, x] : env_val->Map()) { if (not x->IsString()) { (*logger)(fmt::format("env has to evaluate to map of strings, but " "found entry {}", x->ToString()), true); return; } } auto sh_exp = desc->ReadOptionalExpression("sh -c", Expression::kEmptyList); if (not sh_exp) { return; } auto sh_val = sh_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating sh:\n{}", msg), true); }); if (not sh_val) { return; } if (sh_val->IsNone()) { sh_val = Expression::kEmptyList; } if (not sh_val->IsList()) { (*logger)(fmt::format("sh has evaluate to list of strings or null, but " "found {}", sh_val->ToString()), true); return; } for (auto const& entry : sh_val->List()) { if (not entry->IsString()) { (*logger)(fmt::format("sh has evaluate to list of strings or null, " "but found {}\nwith non-string entry {}", sh_val->ToString(), entry->ToString()), true); return; } } static ExpressionPtr const kShC = Expression::FromJson(R"( ["sh", "-c"] )"_json); if (sh_val->List().empty()) { sh_val = kShC; } auto scale_exp = desc->ReadOptionalExpression("timeout scaling", Expression::kOne); if (not scale_exp) { return; } auto scale_val = scale_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating timeout scaling:\n{}", msg), true); }); if (not scale_val) { return; } if (not(scale_val->IsNumber() or scale_val->IsNone())) { (*logger)(fmt::format("timeout scaling has evaluate to a number (or " "null for default), but found {}", scale_val->ToString()), true); return; } auto props_exp = desc->ReadOptionalExpression("execution properties", Expression::kEmptyMapExpr); if (not props_exp) { return; } auto props_val = props_exp.Evaluate( param_config, string_fields_fcts, [&logger](auto const& msg) { (*logger)( fmt::format("While evaluating execution properties:\n{}", msg), true); }); if (not props_val) { return; } if (props_val->IsNone()) { props_val = Expression::kEmptyMap; } if (not props_val->IsMap()) { (*logger)(fmt::format("execution properties has to evaluate to a map " "(or null for default), but found {}", props_val->ToString()), true); return; } for (auto const& [prop_name, prop_val] : props_val->Map()) { if (not prop_val->IsString()) { (*logger)( fmt::format("execution properties has to evaluate to a map (or " "null for default), but found {} for key {}", nlohmann::json(prop_name).dump(), prop_val->ToString()), true); return; } } // Construct inputs; in case of conflicts, artifacts take precedence // over runfiles. auto inputs = ExpressionPtr{Expression::map_t{}}; for (auto const& dep : dependency_values) { inputs = ExpressionPtr{Expression::map_t{inputs, (*dep)->RunFiles()}}; } for (auto const& dep : dependency_values) { inputs = ExpressionPtr{Expression::map_t{inputs, (*dep)->Artifacts()}}; } auto inputs_conflict = BuildMaps::Target::Utils::tree_conflict(inputs); // While syntactical conflicts are resolved in a latest wins (with artifacts // after runfiles), semantic path conflicts are an error. if (inputs_conflict) { (*logger)(fmt::format("Input artifacts have staging conflict on {}", nlohmann::json(*inputs_conflict).dump()), /*fatal=*/true); return; } std::vector trees{}; inputs = BuildMaps::Target::Utils::add_dir_for( cwd_value->String(), inputs, &trees); std::vector argv{}; argv.reserve(sh_val->List().size() + 1); for (auto const& entry : sh_val->List()) { argv.emplace_back(entry->String()); } argv.emplace_back(cmd_ss.str()); // Construct our single action, and its artifacts auto action = BuildMaps::Target::Utils::createAction( outs, out_dirs, argv, cwd_value->String(), env_val, std::nullopt, false, scale_val->IsNumber() ? scale_val->Number() : 1.0, props_val, inputs); auto action_identifier = action->Id(); Expression::map_t::underlying_map_t artifacts; for (const auto& container : {outs, out_dirs}) { for (const auto& path : container) { artifacts.emplace( path, ExpressionPtr{ArtifactDescription::CreateAction( action_identifier, std::filesystem::path{path})}); } } auto artifacts_stage = ExpressionPtr{Expression::map_t{artifacts}}; auto artifacts_conflict = BuildMaps::Target::Utils::tree_conflict(artifacts_stage); if (artifacts_conflict) { (*logger)(fmt::format("artifacts have staging conflicts on {}", nlohmann::json(*artifacts_conflict).dump()), /*fatal=*/true); return; } auto const& empty_map = Expression::kEmptyMap; auto result = std::make_shared( TargetResult{.artifact_stage = std::move(artifacts_stage), .provides = empty_map, .runfiles = empty_map}, std::vector{action}, std::vector{}, std::move(trees), std::vector{}, std::move(effective_vars), std::move(tainted), std::move(implied_export), std::move(deps_info)); result = result_map->Add(key.target, effective_conf, std::move(result)); (*setter)(std::move(result)); } void GenericRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, "generic target", logger); desc->ExpectFields(kGenericRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); auto const& empty_list = Expression::kEmptyList; auto deps_exp = desc->ReadOptionalExpression("deps", empty_list); if (not deps_exp) { return; } auto deps_value = deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) { (*logger)(fmt::format("While evaluating deps:\n{}", msg), true); }); if (not deps_value->IsList()) { (*logger)(fmt::format("Expected deps to evaluate to a list of targets, " "but found {}", deps_value->ToString()), true); return; } std::vector dependency_keys; std::vector transition_keys; dependency_keys.reserve(deps_value->List().size()); transition_keys.reserve(deps_value->List().size()); auto empty_transition = Configuration{Expression::kEmptyMap}; for (auto const& dep_name : deps_value->List()) { auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &dep_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}", dep_name->ToString(), parse_err), true); }); if (not dep_target) { return; } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, key.config}); transition_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{*dep_target, empty_transition}); } (*subcaller)( dependency_keys, [transition_keys = std::move(transition_keys), desc, setter, logger, key, context, result_map](auto const& values) { GenericRuleWithDeps(transition_keys, values, desc, key, context->repo_config, setter, logger, result_map); }, logger); } void ConfigureRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, "configure target", logger); desc->ExpectFields(kConfigureRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; } auto param_config = key.config.Prune(*param_vars); auto configured_target_name_exp = desc->ReadExpression("target"); if (not configured_target_name_exp) { return; } auto configured_target_name = configured_target_name_exp.Evaluate( param_config, {}, [logger](std::string const& msg) { (*logger)( fmt::format("Evaluating 'target' failed with error:\n{}", msg), true); }); if (not configured_target_name) { return; } auto configured_target = BuildMaps::Base::ParseEntityNameFromExpression( configured_target_name, key.target, context->repo_config, [&logger, &configured_target_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing target name {} failed with:\n{}", configured_target_name->ToString(), parse_err), true); }); if (not configured_target) { return; } // Compute and verify taintedness auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, param_config, desc->ReadOptionalExpression("tainted", Expression::kEmptyList), logger); if (not got_tainted) { return; } auto eval_config = desc->ReadOptionalExpression("config", Expression::kEmptyMapExpr); if (not eval_config->IsMap()) { (*logger)(fmt::format("eval_config has to be an expr, but found {}", eval_config->ToString()), true); return; } eval_config = eval_config.Evaluate( param_config, {}, [logger](std::string const& msg) { (*logger)( fmt::format("Evaluating 'config' failed with error:\n{}", msg), true); }); if (not eval_config) { return; } if (not eval_config->IsMap()) { (*logger)(fmt::format("'config' must evaluate to map, but found {}", eval_config->ToString()), true); return; } auto target_config = key.config.Update(eval_config); auto target_to_configure = BuildMaps::Target::ConfiguredTarget{ std::move(*configured_target), std::move(target_config)}; (*subcaller)( {std::move(target_to_configure)}, [setter, logger, vars = std::move(*param_vars), result_map, transition = Configuration{std::move(eval_config)}, tainted = std::move(tainted), key](auto const& values) { auto& configured_target = *values[0]; if (not std::includes(tainted.begin(), tainted.end(), configured_target->Tainted().begin(), configured_target->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } std::unordered_set vars_set{}; for (auto const& x : configured_target->Vars()) { if (not transition.VariableFixed(x)) { vars_set.insert(x); } } vars_set.insert(vars.begin(), vars.end()); auto effective_conf = key.config.Prune(vars_set); auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{key.target, effective_conf}), {configured_target->GraphInformation().Node()}, {}, {}}; auto analysis_result = std::make_shared( configured_target->Result(), std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::move(vars_set), tainted, configured_target->ImpliedExport(), std::move(deps_info)); analysis_result = result_map->Add(key.target, std::move(effective_conf), std::move(analysis_result)); (*setter)(std::move(analysis_result)); }, logger); } auto const kBuiltIns = std::unordered_map< std::string, std::function&, const nlohmann::json&, const BuildMaps::Target::ConfiguredTarget&, const BuildMaps::Target::TargetMap::SubCallerPtr&, const BuildMaps::Target::TargetMap::SetterPtr&, const BuildMaps::Target::TargetMap::LoggerPtr&, const gsl::not_null&)>>{ {"export", ExportRule}, {"file_gen", FileGenRule}, {"tree", TreeRule}, {"tree_overlay", TreeOverlayRule}, {"disjoint_tree_overlay", DisjointTreeOverlayRule}, {"symlink", SymlinkRule}, {"generic", GenericRule}, {"install", InstallRule}, {"configure", ConfigureRule}}; } // namespace namespace BuildMaps::Target { auto IsBuiltInRule(nlohmann::json const& rule_type) -> bool { if (not rule_type.is_string()) { // Names for built-in rules are always strings return false; } auto rule_name = rule_type.get(); return kBuiltIns.contains(rule_name); } auto HandleBuiltin(const gsl::not_null& context, const nlohmann::json& rule_type, const nlohmann::json& desc, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) -> bool { if (not rule_type.is_string()) { // Names for built-in rules are always strings return false; } auto rule_name = rule_type.get(); auto it = kBuiltIns.find(rule_name); if (it == kBuiltIns.end()) { return false; } auto target_logger = std::make_shared( [logger, rule_name, key](auto msg, auto fatal) { (*logger)(fmt::format( "While evaluating {} target {}:\n{}", rule_name, key.ToShortString(Evaluator::GetExpressionLogLimit()), msg), fatal); }); (it->second)( context, desc, key, subcaller, setter, target_logger, result_map); return true; } } // namespace BuildMaps::Target just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/built_in_rules.hpp000066400000000000000000000033761516554100600327130ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_BUILT_IN_RULES_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_BUILT_IN_RULES_HPP #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/main/analyse_context.hpp" namespace BuildMaps::Target { auto HandleBuiltin(const gsl::not_null& context, const nlohmann::json& rule_type, const nlohmann::json& desc, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) -> bool; } // namespace BuildMaps::Target #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_BUILT_IN_RULES_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/configured_target.hpp000066400000000000000000000051631516554100600333630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_CONFIGURED_TARGET_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_CONFIGURED_TARGET_HPP #include #include #include #include #include "fmt/core.h" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/json.hpp" namespace BuildMaps::Target { struct ConfiguredTarget { BuildMaps::Base::EntityName target; Configuration config; [[nodiscard]] auto operator==(BuildMaps::Target::ConfiguredTarget const& other) const noexcept -> bool { return target == other.target and config == other.config; } [[nodiscard]] auto ToString() const noexcept -> std::string { return fmt::format("[{},{}]", target.ToString(), config.ToString()); } [[nodiscard]] auto ToShortString(std::size_t config_length) const noexcept -> std::string { return fmt::format( "[{},{}]", target.ToString(), AbbreviateJson(PruneJson(config.ToJson()), config_length)); } [[nodiscard]] auto operator<(ConfiguredTarget const& other) const noexcept -> bool { if (target < other.target) { return true; } if (other.target < target) { return false; } return config < other.config; } }; using ConfiguredTargetPtr = std::shared_ptr; } // namespace BuildMaps::Target namespace std { template <> struct hash { [[nodiscard]] auto operator()(BuildMaps::Target::ConfiguredTarget const& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine<>(&seed, ct.target); hash_combine<>(&seed, ct.config); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_CONFIGURED_TARGET_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/export.cpp000066400000000000000000000357241516554100600312120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/export.hpp" #include #include #include #include #include #include #include #include // std::move #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/json.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/serve_api.hpp" #endif // BOOTSTRAP_BUILD_TOOL namespace { auto const kExpectedFields = std::unordered_set{"config_doc", "doc", "fixed_config", "flexible_config", "target", "type"}; void FinalizeExport( const std::vector& exported, const BuildMaps::Base::EntityName& target, const std::vector& vars, const Configuration& effective_config, const std::optional& target_cache_key, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const BuildMaps::Target::TargetMap::SetterPtr& setter, const gsl::not_null& result_map) { const auto* value = exported[0]; if (not(*value)->Tainted().empty()) { (*logger)("Only untainted targets can be exported.", true); return; } auto provides = (*value)->Provides(); if (not provides->IsCacheable()) { (*logger)(fmt::format("Only cacheable values can be exported; but " "target provides {}", provides->ToString()), true); return; } auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{.target = target, .config = effective_config}), {(*value)->GraphInformation().Node()}, {}, {}}; std::unordered_set vars_set{}; vars_set.insert(vars.begin(), vars.end()); std::set implied{}; if (target_cache_key) { implied.insert((*value)->ImpliedExport().begin(), (*value)->ImpliedExport().end()); implied.insert(target_cache_key->Id().digest.hash()); } auto analysis_result = std::make_shared( TargetResult{.artifact_stage = (*value)->Artifacts(), .provides = provides, .runfiles = (*value)->RunFiles()}, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::move(vars_set), std::set{}, std::move(implied), std::move(deps_info)); analysis_result = result_map->Add(target, effective_config, std::move(analysis_result), target_cache_key, true); (*setter)(std::move(analysis_result)); } } // namespace void ExportRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( desc_json, key.target, "export target", logger); auto flexible_vars = desc->ReadStringList("flexible_config"); if (not flexible_vars) { return; } auto effective_config = key.config.Prune(*flexible_vars); if (key.config != effective_config) { (*subcaller)( {BuildMaps::Target::ConfiguredTarget{.target = key.target, .config = effective_config}}, [setter](auto const& values) { AnalysedTargetPtr result = *(values[0]); (*setter)(std::move(result)); }, logger); return; } context->statistics->IncrementExportsFoundCounter(); std::optional target_cache_key; #ifndef BOOTSTRAP_BUILD_TOOL auto const& target_name = key.target.GetNamedTarget(); auto repo_key = context->repo_config->RepositoryKey(*context->storage, target_name.repository); target_cache_key = repo_key ? context->storage->TargetCache().ComputeKey( *repo_key, target_name, effective_config) : std::nullopt; #endif if (target_cache_key) { // first try to get value from local target cache auto target_cache_value = context->storage->TargetCache().Read(*target_cache_key); bool from_just_serve{false}; #ifndef BOOTSTRAP_BUILD_TOOL // if not found locally, try the serve endpoint if (not target_cache_value and context->serve != nullptr) { Logger::Log(LogLevel::Debug, "Querying serve endpoint for export target {}", key.target.ToString()); auto task = fmt::format("[{},{}]", key.target.ToString(), PruneJson(effective_config.ToJson()).dump()); context->progress->TaskTracker().Start(task); auto res = context->serve->ServeTarget(*target_cache_key, *repo_key); // process response from serve endpoint if (not res) { // target not found: log to performance, and continue Logger::Log(LogLevel::Performance, "Export target {} not known to serve endpoint", key.target.ToString()); } else { switch (res->index()) { case 0: { // target found but failed to analyse/build: this should // be a fatal error for the local build too (*logger)( fmt::format( "Failure to remotely analyse or build target " "{}\nDetailed log available on the " "remote-execution endpoint as blob {}", key.target.ToString(), std::get<0>(*res)), /*fatal=*/true); return; } case 1: { // internal failure on the serve endpoint, or failures // on the client side: local build should not continue (*logger)(fmt::format("While querying serve endpoint " "for export target {}:\n{}", key.target.ToString(), std::get<1>(*res)), /*fatal=*/true); return; } case 2: { // some other failure occurred on the serve endpoint; // log to debug and continue locally Logger::Log(LogLevel::Debug, "While querying serve endpoint for export " "target {}:\n{}", key.target.ToString(), std::get<2>(*res)); } break; default: { // index == 3 target_cache_value = std::get<3>(*res); from_just_serve = true; } } } context->progress->TaskTracker().Stop(task); } #endif // BOOTSTRAP_BUILD_TOOL if (not target_cache_value) { context->statistics->IncrementExportsUncachedCounter(); Logger::Log(LogLevel::Performance, "Export target {} registered for caching: {}", key.target.ToString(), target_cache_key->Id().ToString()); } else { auto const& [entry, info] = *target_cache_value; if (auto result = entry.ToResult()) { auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{ .target = key.target, .config = effective_config}), {}, {}, {}}; auto analysis_result = std::make_shared( *result, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{flexible_vars->begin(), flexible_vars->end()}, std::set{}, entry.ToImplied(), deps_info); analysis_result = result_map->Add(key.target, effective_config, std::move(analysis_result), std::nullopt, true); Logger::Log(LogLevel::Performance, "Export target {} taken from {}: {} -> {}", key.target.ToString(), (from_just_serve ? "serve endpoint" : "cache"), target_cache_key->Id().ToString(), info.ToString()); (*setter)(std::move(analysis_result)); if (from_just_serve) { context->statistics->IncrementExportsServedCounter(); } else { context->statistics->IncrementExportsCachedCounter(); } return; } (*logger)(fmt::format("Reading target entry for key {} failed", target_cache_key->Id().ToString()), false); } } else { context->statistics->IncrementExportsNotEligibleCounter(); Logger::Log(LogLevel::Performance, "Export target {} is not eligible for target caching", key.target.ToString()); } desc->ExpectFields(kExpectedFields); auto exported_target_name = desc->ReadExpression("target"); if (not exported_target_name) { return; } auto exported_target = BuildMaps::Base::ParseEntityNameFromExpression( exported_target_name, key.target, context->repo_config, [&logger, &exported_target_name](std::string const& parse_err) { (*logger)(fmt::format("Parsing target name {} failed with:\n{}", exported_target_name->ToString(), parse_err), true); }); if (not exported_target) { return; } auto fixed_config = desc->ReadOptionalExpression("fixed_config", Expression::kEmptyMap); if (not fixed_config->IsMap()) { (*logger)(fmt::format("fixed_config has to be a map, but found {}", fixed_config->ToString()), true); return; } for (auto const& var : fixed_config->Map().Keys()) { if (effective_config.VariableFixed(var)) { (*logger)( fmt::format("Variable {} is both fixed and flexible.", var), true); return; } } auto target_config = effective_config.Update(fixed_config); (*subcaller)( {BuildMaps::Target::ConfiguredTarget{ .target = std::move(*exported_target), .config = std::move(target_config)}}, [setter, logger, vars = std::move(*flexible_vars), result_map, effective_config = std::move(effective_config), target_cache_key = std::move(target_cache_key), target = key.target](auto const& values) { FinalizeExport(values, target, vars, effective_config, target_cache_key, logger, setter, result_map); }, logger); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/export.hpp000066400000000000000000000030011516554100600311760ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_EXPORT_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_EXPORT_HPP #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/main/analyse_context.hpp" void ExportRule( const gsl::not_null& context, const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map); #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_EXPORT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/result_map.hpp000066400000000000000000000505051516554100600320430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/storage/target_cache_key.hpp" namespace BuildMaps::Target { // Class collecting analysed targets for their canonical configuration. class ResultTargetMap { public: struct ActionWithOrigin { ActionDescription::Ptr desc; nlohmann::json origin; }; template struct ResultType { std::vector actions; std::vector blobs; std::vector trees; std::vector tree_overlays; }; explicit ResultTargetMap(std::size_t jobs) : width_{ComputeWidth(jobs)} {} ResultTargetMap() = default; // \brief Add the analysed target for the given target and // configuration, if no entry is present for the given // target-configuration pair. \returns the analysed target that is // element of the map after insertion. [[nodiscard]] auto Add( BuildMaps::Base::EntityName name, Configuration conf, gsl::not_null const& result, std::optional target_cache_key = std::nullopt, bool is_export_target = false) -> AnalysedTargetPtr { auto part = std::hash{}(name) % width_; std::unique_lock lock{m_[part]}; auto [entry, inserted] = targets_[part].emplace(ConfiguredTarget{.target = std::move(name), .config = std::move(conf)}, result); if (target_cache_key) { cache_targets_[part].emplace(*target_cache_key, entry->second); } if (is_export_target) { export_targets_[part].emplace(entry->first); } if (inserted) { num_actions_[part] += entry->second->Actions().size(); num_blobs_[part] += entry->second->Blobs().size(); num_trees_[part] += entry->second->Trees().size(); num_tree_overlays_[part] += entry->second->TreeOverlays().size(); } return entry->second; } [[nodiscard]] auto ConfiguredTargets() const noexcept -> std::vector { std::vector targets{}; std::size_t s = 0; for (const auto& target : targets_) { s += target.size(); } targets.reserve(s); for (const auto& i : targets_) { std::transform(i.begin(), i.end(), std::back_inserter(targets), [](auto const& target) { return target.first; }); } std::sort(targets.begin(), targets.end(), [](auto const& lhs, auto const& rhs) { return lhs.ToString() < rhs.ToString(); }); return targets; } [[nodiscard]] auto ExportTargets() const noexcept -> std::vector { std::vector all_exports{}; for (auto const& exports : export_targets_) { all_exports.insert( all_exports.end(), exports.begin(), exports.end()); } std::sort(all_exports.begin(), all_exports.end(), [](auto const& lhs, auto const& rhs) { return lhs.ToString() < rhs.ToString(); }); return all_exports; } [[nodiscard]] auto ConfiguredTargetsGraph() const -> nlohmann::json { auto result = nlohmann::json::object(); for (auto const& i : targets_) { for (auto const& it : i) { auto const& info = it.second->GraphInformation(); auto node = info.NodeString(); if (node) { result[*node] = info.DepsToJson(); } } } return result; } [[nodiscard]] auto CacheTargets() const noexcept -> std::unordered_map { return std::accumulate( cache_targets_.begin(), cache_targets_.end(), std::unordered_map{}, [](auto&& l, auto const& r) { l.insert(r.begin(), r.end()); return std::forward(l); }); } [[nodiscard]] auto GetAction(const ActionIdentifier& identifier) -> std::optional { for (const auto& target : targets_) { for (const auto& el : target) { for (const auto& action : el.second->Actions()) { if (action->Id() == identifier) { return action; } } } } return std::nullopt; } template [[nodiscard]] auto ToResult(gsl::not_null const& stats, gsl::not_null const& progress, Logger const* logger = nullptr) const -> ResultType { ResultType result{}; std::size_t na = 0; std::size_t nb = 0; std::size_t nt = 0; std::size_t nto = 0; for (std::size_t i = 0; i < width_; i++) { na += num_actions_[i]; nb += num_blobs_[i]; nt += num_trees_[i]; nto += num_tree_overlays_[i]; } result.actions.reserve(na); result.blobs.reserve(nb); result.trees.reserve(nt); result.tree_overlays.reserve(nto); auto& origin_map = progress->OriginMap(); origin_map.clear(); origin_map.reserve(na + nto); for (const auto& target : targets_) { std::for_each(target.begin(), target.end(), [&](auto const& el) { auto const& actions = el.second->Actions(); std::size_t pos{}; std::for_each( actions.begin(), actions.end(), [&origin_map, &pos, &el](auto const& action) { std::pair origin{ el.first, pos++}; auto id = action->Id(); if (origin_map.contains(id)) { origin_map[id].push_back(origin); } else { origin_map[id] = std::vector< std::pair>{ origin}; } }); auto const& tree_overlays = el.second->TreeOverlays(); std::for_each( tree_overlays.begin(), tree_overlays.end(), [&origin_map, &pos, &el](auto const& overlay) { std::pair origin{ el.first, pos++}; auto id = overlay->Id(); if (origin_map.contains(id)) { origin_map[id].push_back(origin); } else { origin_map[id] = std::vector< std::pair>{ origin}; } }); }); // Sort origins to get a reproducible order. We don't expect many // origins for a single action, so the cost of comparison is not // too important. Moreover, we expect most actions to have a single // origin, so any precomputation would be more expensive. for (auto const& i : origin_map) { std::sort(origin_map[i.first].begin(), origin_map[i.first].end(), [](auto const& left, auto const& right) { auto const& left_target = left.first; auto const& right_target = right.first; return (left_target < right_target) or (left_target == right_target and left.second < right.second); }); } } for (const auto& target : targets_) { std::for_each(target.begin(), target.end(), [&](auto const& el) { auto const& actions = el.second->Actions(); if constexpr (kIncludeOrigins) { std::for_each( actions.begin(), actions.end(), [&result, &origin_map](auto const& action) { auto origins = nlohmann::json::array(); for (auto const& [ct, count] : origin_map[action->Id()]) { origins.push_back(nlohmann::json{ {"target", ct.target.ToJson()}, {"subtask", count}, {"config", ct.config.ToJson()}}); } result.actions.emplace_back(ActionWithOrigin{ .desc = action, .origin = origins}); }); } else { std::for_each(actions.begin(), actions.end(), [&result](auto const& action) { result.actions.emplace_back(action); }); } auto const& blobs = el.second->Blobs(); auto const& trees = el.second->Trees(); auto const& tree_overlays = el.second->TreeOverlays(); result.blobs.insert( result.blobs.end(), blobs.begin(), blobs.end()); result.trees.insert( result.trees.end(), trees.begin(), trees.end()); result.tree_overlays.insert(result.tree_overlays.end(), tree_overlays.begin(), tree_overlays.end()); }); } std::sort(result.blobs.begin(), result.blobs.end()); auto lastblob = std::unique(result.blobs.begin(), result.blobs.end()); result.blobs.erase(lastblob, result.blobs.end()); std::sort( result.trees.begin(), result.trees.end(), [](auto left, auto right) { return left->Id() < right->Id(); }); auto lasttree = std::unique( result.trees.begin(), result.trees.end(), [](auto left, auto right) { return left->Id() == right->Id(); }); result.trees.erase(lasttree, result.trees.end()); std::sort( result.tree_overlays.begin(), result.tree_overlays.end(), [](auto left, auto right) { return left->Id() < right->Id(); }); auto lasttree_overlay = std::unique( result.tree_overlays.begin(), result.tree_overlays.end(), [](auto left, auto right) { return left->Id() == right->Id(); }); result.tree_overlays.erase(lasttree_overlay, result.tree_overlays.end()); std::sort(result.actions.begin(), result.actions.end(), [](auto left, auto right) { if constexpr (kIncludeOrigins) { return left.desc->Id() < right.desc->Id(); } else { return left->Id() < right->Id(); } }); auto lastaction = std::unique(result.actions.begin(), result.actions.end(), [](auto left, auto right) { if constexpr (kIncludeOrigins) { return left.desc->Id() == right.desc->Id(); } else { return left->Id() == right->Id(); } }); result.actions.erase(lastaction, result.actions.end()); auto& output_map = progress->OutputMap(); output_map.clear(); output_map.reserve(na); if constexpr (kIncludeOrigins) { for (auto const& action : result.actions) { if (not action.desc->OutputFiles().empty()) { output_map[action.desc->Id()] = action.desc->OutputFiles()[0]; } else if (not action.desc->OutputDirs().empty()) { output_map[action.desc->Id()] = action.desc->OutputDirs()[0]; } } } else { for (auto const& action : result.actions) { if (not action->OutputFiles().empty()) { output_map[action->Id()] = action->OutputFiles()[0]; } else if (not action->OutputDirs().empty()) { output_map[action->Id()] = action->OutputDirs()[0]; } } } int trees_traversed = stats->TreesAnalysedCounter(); if (trees_traversed > 0) { Logger::Log(logger, LogLevel::Performance, "Analysed {} non-known source trees", trees_traversed); } Logger::Log( logger, LogLevel::Info, "Discovered {} actions, {} tree overlays, {} trees, {} blobs", result.actions.size(), result.tree_overlays.size(), result.trees.size(), result.blobs.size()); return result; } template [[nodiscard]] auto ToJson(gsl::not_null const& stats, gsl::not_null const& progress, Logger const* logger = nullptr) const -> nlohmann::json { auto const result = ToResult(stats, progress, logger); auto actions = nlohmann::json::object(); auto trees = nlohmann::json::object(); auto tree_overlays = nlohmann::json::object(); std::for_each(result.actions.begin(), result.actions.end(), [&actions](auto const& action) { if constexpr (kIncludeOrigins) { auto const& id = action.desc->GraphAction().Id(); actions[id] = action.desc->ToJson(); actions[id]["origins"] = action.origin; } else { auto const& id = action->GraphAction().Id(); actions[id] = action->ToJson(); } }); std::for_each( result.trees.begin(), result.trees.end(), [&trees](auto const& tree) { trees[tree->Id()] = tree->ToJson(); }); std::for_each(result.tree_overlays.begin(), result.tree_overlays.end(), [&tree_overlays](auto const& tree_overlay) { tree_overlays[tree_overlay->Id()] = tree_overlay->ToJson(); }); auto json = nlohmann::json{ {"actions", actions}, {"blobs", result.blobs}, {"trees", trees}}; if (not tree_overlays.empty()) { json["tree_overlays"] = tree_overlays; } return json; } template auto ToFile(std::vector const& destinations, gsl::not_null const& stats, gsl::not_null const& progress, int indent = 2) const -> void { auto logger = Logger("result-dumping"); logger.SetLogLimit(LogLevel::Error); if (not destinations.empty()) { // As serialization is expensive, compute the string only once auto data = ToJson(stats, progress, &logger).dump(indent); for (auto const& graph_file : destinations) { Logger::Log(LogLevel::Info, "Dumping action graph to file {}.", graph_file.string()); std::ofstream os(graph_file); os << data << std::endl; } } } void Clear(gsl::not_null const& ts) { for (std::size_t i = 0; i < width_; ++i) { ts->QueueTask([i, this]() { targets_[i].clear(); cache_targets_[i].clear(); }); } } private: constexpr static std::size_t kScalingFactor = 2; std::size_t width_{ComputeWidth(0)}; std::vector m_{width_}; std::vector< std::unordered_map>> targets_{width_}; std::vector< std::unordered_map>> cache_targets_{width_}; std::vector> export_targets_{width_}; std::vector num_actions_{std::vector(width_)}; std::vector num_blobs_{std::vector(width_)}; std::vector num_trees_{std::vector(width_)}; std::vector num_tree_overlays_{ std::vector(width_)}; constexpr static auto ComputeWidth(std::size_t jobs) -> std::size_t { if (jobs <= 0) { // Non-positive indicates to use the default value return ComputeWidth( std::max(1U, std::thread::hardware_concurrency())); } return jobs * kScalingFactor + 1; } }; // namespace BuildMaps::Target template <> struct ResultTargetMap::ResultType { std::vector actions; std::vector blobs; std::vector trees; std::vector tree_overlays; }; } // namespace BuildMaps::Target #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/target_map.cpp000066400000000000000000002703621516554100600320130ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/target_map.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_function.hpp" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/base_maps/user_rule.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/build_engine/expression/target_node.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/build_engine/target_map/built_in_rules.hpp" #include "src/buildtool/build_engine/target_map/utils.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/vector.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/serve_api.hpp" #endif // BOOTSTRAP_BUILD_TOOL namespace { using namespace std::string_literals; [[nodiscard]] auto ReadActionOutputExpr(ExpressionPtr const& out_exp, std::string const& field_name) -> ActionDescription::outputs_t { if (not out_exp->IsList()) { throw Evaluator::EvaluationError{ fmt::format("{} has to be a list of strings, but found {}", field_name, out_exp->ToString())}; } ActionDescription::outputs_t outputs; outputs.reserve(out_exp->List().size()); for (auto const& out_path : out_exp->List()) { if (not out_path->IsString()) { throw Evaluator::EvaluationError{ fmt::format("{} has to be a list of strings, but found {}", field_name, out_exp->ToString())}; } outputs.emplace_back(out_path->String()); } return outputs; } struct TargetData { using Ptr = std::shared_ptr; std::vector target_vars; std::unordered_map config_exprs; std::unordered_map string_exprs; std::unordered_map target_exprs; ExpressionPtr tainted_expr; bool parse_target_names{}; TargetData(std::vector target_vars, std::unordered_map config_exprs, std::unordered_map string_exprs, std::unordered_map target_exprs, ExpressionPtr tainted_expr, bool parse_target_names) : target_vars{std::move(target_vars)}, config_exprs{std::move(config_exprs)}, string_exprs{std::move(string_exprs)}, target_exprs{std::move(target_exprs)}, tainted_expr{std::move(tainted_expr)}, parse_target_names{parse_target_names} {} [[nodiscard]] static auto FromFieldReader( BuildMaps::Base::UserRulePtr const& rule, BuildMaps::Base::FieldReader::Ptr const& desc) -> TargetData::Ptr { desc->ExpectFields(rule->ExpectedFields()); auto target_vars = desc->ReadStringList("arguments_config"); auto tainted_expr = desc->ReadOptionalExpression("tainted", Expression::kEmptyList); auto convert_to_exprs = [&desc](gsl::not_null< std::unordered_map*> const& expr_map, std::vector const& field_names) -> bool { for (auto const& field_name : field_names) { auto expr = desc->ReadOptionalExpression( field_name, Expression::kEmptyList); if (not expr) { return false; } expr_map->emplace(field_name, std::move(expr)); } return true; }; std::unordered_map config_exprs; std::unordered_map string_exprs; std::unordered_map target_exprs; if (target_vars and tainted_expr and convert_to_exprs(&config_exprs, rule->ConfigFields()) and convert_to_exprs(&string_exprs, rule->StringFields()) and convert_to_exprs(&target_exprs, rule->TargetFields())) { return std::make_shared(std::move(*target_vars), std::move(config_exprs), std::move(string_exprs), std::move(target_exprs), std::move(tainted_expr), /*parse_target_names=*/true); } return nullptr; } [[nodiscard]] static auto FromTargetNode( BuildMaps::Base::UserRulePtr const& rule, TargetNode::Abstract const& node, ExpressionPtr const& rule_map, gsl::not_null const& logger) -> TargetData::Ptr { auto const& string_fields = node.string_fields->Map(); auto const& target_fields = node.target_fields->Map(); std::unordered_map config_exprs; std::unordered_map string_exprs; std::unordered_map target_exprs; for (auto const& field_name : rule->ConfigFields()) { if (target_fields.Find(field_name)) { (*logger)( fmt::format( "Expected config field '{}' in string_fields of " "abstract node type '{}', and not in target_fields", field_name, node.node_type), /*fatal=*/true); return nullptr; } auto const& config_expr = *(string_fields.Find(field_name) .value_or(&Expression::kEmptyList)); config_exprs.emplace(field_name, config_expr); } for (auto const& field_name : rule->StringFields()) { if (target_fields.Find(field_name)) { (*logger)( fmt::format( "Expected string field '{}' in string_fields of " "abstract node type '{}', and not in target_fields", field_name, node.node_type), /*fatal=*/true); return nullptr; } auto const& string_expr = *(string_fields.Find(field_name) .value_or(&Expression::kEmptyList)); string_exprs.emplace(field_name, string_expr); } for (auto const& field_name : rule->TargetFields()) { if (string_fields.Find(field_name)) { (*logger)( fmt::format( "Expected target field '{}' in target_fields of " "abstract node type '{}', and not in string_fields", field_name, node.node_type), /*fatal=*/true); return nullptr; } auto const& target_expr = *(target_fields.Find(field_name) .value_or(&Expression::kEmptyList)); auto const& nodes = target_expr->List(); Expression::list_t targets{}; targets.reserve(nodes.size()); for (auto const& node_expr : nodes) { targets.emplace_back(BuildMaps::Base::EntityName{ BuildMaps::Base::AnonymousTarget{ .rule_map = rule_map, .target_node = node_expr}}); } target_exprs.emplace(field_name, targets); } return std::make_shared(std::vector{}, std::move(config_exprs), std::move(string_exprs), std::move(target_exprs), Expression::kEmptyList, /*parse_target_names=*/false); } }; auto NameTransitionedDeps( const BuildMaps::Target::ConfiguredTarget& transitioned_target, const AnalysedTargetPtr& analysis, const Configuration& effective_conf) -> std::string { auto conf = effective_conf.Update(transitioned_target.config.Expr()) .Prune(analysis->Vars()); return BuildMaps::Target::ConfiguredTarget{transitioned_target.target, conf} .ToShortString(Evaluator::GetExpressionLogLimit()); } // Check if an object is contained an expression; to avoid tree-unfolding // the expression, we need to cache the values already computed. auto ExpressionContainsObject(std::unordered_map* map, const ExpressionPtr& object, const ExpressionPtr& exp) { auto lookup = map->find(exp); if (lookup != map->end()) { return lookup->second; } auto result = false; if (exp == object) { result = true; } else if (exp->IsList()) { for (auto const& entry : exp->List()) { if (ExpressionContainsObject(map, object, entry)) { result = true; break; } } } else if (exp->IsMap()) { for (auto const& [k, v] : exp->Map()) { if (ExpressionContainsObject(map, object, v)) { result = true; break; } } } map->insert({exp, result}); return result; } auto ListDependencies( const ExpressionPtr& object, const std::unordered_map& deps_by_transition, const Configuration& effective_conf) -> std::string { std::stringstream deps{}; std::unordered_map contains_object{}; for (auto const& [transition_target, analysis] : deps_by_transition) { for (auto const& [path, value] : analysis->Artifacts().Map()) { if (value == object) { deps << fmt::format( "\n - {}, artifact at {}", NameTransitionedDeps( transition_target, analysis, effective_conf), nlohmann::json(path).dump()); } } for (auto const& [path, value] : analysis->RunFiles().Map()) { if (value == object) { deps << fmt::format( "\n - {}, runfile at {}", NameTransitionedDeps( transition_target, analysis, effective_conf), nlohmann::json(path).dump()); } } if (ExpressionContainsObject( &contains_object, object, analysis->Provides())) { deps << fmt::format( "\n - {}, in provided data", NameTransitionedDeps( transition_target, analysis, effective_conf)); } } return deps.str(); } auto ExprToTree(std::string const& context, ExpressionPtr const& val) -> Tree::Ptr { if (not val->IsMap()) { throw Evaluator::EvaluationError{ fmt::format("{} has to be a map of artifacts, " "but found {}", context, val->ToString())}; } std::unordered_map artifacts; artifacts.reserve(val->Map().size()); for (auto const& [input_path, artifact] : val->Map()) { if (not artifact->IsArtifact()) { throw Evaluator::EvaluationError{ fmt::format("{} has to be a map of artifacts, " "but found {} for {}", artifact->ToString(), input_path, context)}; } auto norm_path = ToNormalPath(std::filesystem::path{input_path}); artifacts.emplace(std::move(norm_path), artifact->Artifact()); } auto conflict = BuildMaps::Target::Utils::tree_conflict(val); if (conflict) { throw Evaluator::EvaluationError{fmt::format( "Tree conflicts on subtree {} of {}", *conflict, context)}; } auto tree = std::make_shared(std::move(artifacts)); return tree; } void withDependencies( const gsl::not_null& context, const std::vector& transition_keys, const std::vector& dependency_values, std::size_t declared_count, std::size_t declared_and_implicit_count, const BuildMaps::Base::UserRulePtr& rule, const TargetData::Ptr& data, const BuildMaps::Target::ConfiguredTarget& key, std::unordered_map params, // NOLINT const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map) { // Associate dependency keys with values std::unordered_map deps_by_transition; deps_by_transition.reserve(transition_keys.size()); ExpectsAudit(transition_keys.size() == dependency_values.size()); for (std::size_t i = 0; i < transition_keys.size(); ++i) { deps_by_transition.emplace(transition_keys[i], *dependency_values[i]); } // Compute the effective dependency on config variables std::unordered_set effective_vars; auto const& param_vars = data->target_vars; effective_vars.insert(param_vars.begin(), param_vars.end()); auto const& config_vars = rule->ConfigVars(); effective_vars.insert(config_vars.begin(), config_vars.end()); for (auto const& [transition, target] : deps_by_transition) { for (auto const& x : target->Vars()) { if (not transition.config.VariableFixed(x)) { effective_vars.insert(x); } } } auto effective_conf = key.config.Prune(effective_vars); std::vector declared_deps{}; std::vector implicit_deps{}; std::vector anonymous_deps{}; ExpectsAudit(declared_count <= declared_and_implicit_count); ExpectsAudit(declared_and_implicit_count <= dependency_values.size()); auto fill_target_graph = [&dependency_values](std::size_t const a, std::size_t const b, auto* deps) { std::transform( dependency_values.begin() + static_cast(a), dependency_values.begin() + static_cast(b), std::back_inserter(*deps), [](auto dep) { return (*(dep))->GraphInformation().Node(); }); }; fill_target_graph(0, declared_count, &declared_deps); fill_target_graph( declared_count, declared_and_implicit_count, &implicit_deps); fill_target_graph( declared_and_implicit_count, dependency_values.size(), &anonymous_deps); auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{.target = key.target, .config = effective_conf}), declared_deps, implicit_deps, anonymous_deps}; // Compute and verify taintedness auto tainted = std::set{}; auto got_tainted = BuildMaps::Target::Utils::getTainted( &tainted, key.config.Prune(param_vars), data->tainted_expr, logger); if (not got_tainted) { return; } tainted.insert(rule->Tainted().begin(), rule->Tainted().end()); for (auto const& dep : dependency_values) { if (not std::includes(tainted.begin(), tainted.end(), (*dep)->Tainted().begin(), (*dep)->Tainted().end())) { (*logger)( "Not tainted with all strings the dependencies are tainted " "with", true); return; } } // Compute implied export targets std::set implied_export{}; for (auto const& dep : dependency_values) { implied_export.insert((*dep)->ImpliedExport().begin(), (*dep)->ImpliedExport().end()); } // Evaluate string parameters auto string_fields_fcts = FunctionMap::MakePtr(FunctionMap::underlying_map_t{ {"outs", [&deps_by_transition, &key, context]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, context->repo_config, deps_by_transition) ->Artifacts()); }}, {"runfiles", [&deps_by_transition, &key, context]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::keys_expr( BuildMaps::Target::Utils::obtainTargetByName( eval, expr, env, key.target, context->repo_config, deps_by_transition) ->RunFiles()); }}}); auto param_config = key.config.Prune(param_vars); params.reserve(params.size() + rule->StringFields().size()); for (auto const& field_name : rule->StringFields()) { auto const& field_exp = data->string_exprs[field_name]; auto field_value = field_exp.Evaluate( param_config, string_fields_fcts, [&logger, &field_name](auto const& msg) { (*logger)(fmt::format("While evaluating string field {}:\n{}", field_name, msg), true); }); if (not field_value) { return; } if (not field_value->IsList()) { (*logger)(fmt::format("String field {} should be a list of " "strings, but found {}", field_name, field_value->ToString()), true); return; } for (auto const& entry : field_value->List()) { if (not entry->IsString()) { (*logger)(fmt::format("String field {} should be a list of " "strings, but found entry {}", field_name, entry->ToString()), true); return; } } params.emplace(field_name, std::move(field_value)); } // Evaluate main expression auto expression_config = key.config.Prune(config_vars); std::vector actions{}; std::vector blobs{}; std::vector trees{}; std::vector tree_overlays{}; auto main_exp_fcts = FunctionMap::MakePtr( {{"FIELD", [¶ms](auto&& eval, auto const& expr, auto const& env) { auto name = eval(expr["name"], env); if (not name->IsString()) { throw Evaluator::EvaluationError{ fmt::format("FIELD argument 'name' should evaluate to a " "string, but got {}", name->ToString())}; } auto it = params.find(name->String()); if (it == params.end()) { throw Evaluator::EvaluationError{ fmt::format("FIELD '{}' unknown", name->String())}; } return it->second; }}, {"DEP_ARTIFACTS", [&deps_by_transition]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::obtainTarget( eval, expr, env, deps_by_transition) ->Artifacts(); }}, {"DEP_RUNFILES", [&deps_by_transition]( auto&& eval, auto const& expr, auto const& env) { return BuildMaps::Target::Utils::obtainTarget( eval, expr, env, deps_by_transition) ->RunFiles(); }}, {"DEP_PROVIDES", [&deps_by_transition]( auto&& eval, auto const& expr, auto const& env) { auto const& provided = BuildMaps::Target::Utils::obtainTarget( eval, expr, env, deps_by_transition) ->Provides(); auto provider = eval(expr["provider"], env); auto provided_value = provided->At(provider->String()); if (provided_value) { return provided_value->get(); } auto const& empty_list = Expression::kEmptyList; return eval(expr->Get("default", empty_list), env); }}, {"ACTION", [&actions, &rule, &trees]( auto&& eval, auto const& expr, auto const& env) { auto const& empty_map_exp = Expression::kEmptyMapExpr; auto inputs_exp = eval(expr->Get("inputs", empty_map_exp), env); if (not inputs_exp->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "inputs has to be a map of artifacts, but found {}", inputs_exp->ToString())}; } for (auto const& [input_path, artifact] : inputs_exp->Map()) { if (not artifact->IsArtifact()) { throw Evaluator::EvaluationError{ fmt::format("inputs has to be a map of Artifacts, " "but found {} for {}", artifact->ToString(), input_path)}; } } auto conflict_or_normal = BuildMaps::Target::Utils::artifacts_tree(inputs_exp); if (std::holds_alternative(conflict_or_normal)) { throw Evaluator::EvaluationError{ fmt::format("inputs conflict on path {}", std::get(conflict_or_normal))}; } inputs_exp = std::get(conflict_or_normal); auto conflict = BuildMaps::Target::Utils::tree_conflict(inputs_exp); if (conflict) { throw Evaluator::EvaluationError{ fmt::format("inputs conflicts on subtree {}", *conflict)}; } Expression::map_t::underlying_map_t result; auto outputs = ReadActionOutputExpr( eval(expr->Get("outs", Expression::list_t{}), env), "outs"); auto output_dirs = ReadActionOutputExpr( eval(expr->Get("out_dirs", Expression::list_t{}), env), "out_dirs"); if (outputs.empty() and output_dirs.empty()) { throw Evaluator::EvaluationError{ "either outs or out_dirs must be specified for ACTION"}; } ActionDescription::outputs_t outputs_norm{}; ActionDescription::outputs_t output_dirs_norm{}; outputs_norm.reserve(outputs.size()); output_dirs_norm.reserve(output_dirs.size()); std::for_each( outputs.begin(), outputs.end(), [&outputs_norm](auto p) { outputs_norm.emplace_back(ToNormalPath(p)); }); std::for_each(output_dirs.begin(), output_dirs.end(), [&output_dirs_norm](auto p) { output_dirs_norm.emplace_back(ToNormalPath(p)); }); sort_and_deduplicate(&outputs_norm); sort_and_deduplicate(&output_dirs_norm); // find entries present on both fields std::vector dups{}; std::set_intersection(outputs_norm.begin(), outputs_norm.end(), output_dirs_norm.begin(), output_dirs_norm.end(), std::back_inserter(dups)); if (not dups.empty()) { throw Evaluator::EvaluationError{ fmt::format("outs and out_dirs for ACTION must be " "disjoint. Found repeated entries:\n{}", nlohmann::json(dups).dump())}; } std::vector cmd; auto cmd_exp = eval(expr->Get("cmd", Expression::list_t{}), env); if (not cmd_exp->IsList()) { throw Evaluator::EvaluationError{fmt::format( "cmd has to be a list of strings, but found {}", cmd_exp->ToString())}; } if (cmd_exp->List().empty()) { throw Evaluator::EvaluationError{ "cmd must not be an empty list"}; } cmd.reserve(cmd_exp->List().size()); for (auto const& arg : cmd_exp->List()) { if (not arg->IsString()) { throw Evaluator::EvaluationError{fmt::format( "cmd has to be a list of strings, but found {}", cmd_exp->ToString())}; } cmd.emplace_back(arg->String()); } auto cwd_exp = eval(expr->Get("cwd", Expression::kEmptyString), env); if (not cwd_exp->IsString()) { throw Evaluator::EvaluationError{ fmt::format("cwd has to be a string, but found {}", cwd_exp->ToString())}; } if (not PathIsNonUpwards(cwd_exp->String())) { throw Evaluator::EvaluationError{fmt::format( "cwd has to be a non-upwards relative path, but found {}", cwd_exp->ToString())}; } auto final_inputs = BuildMaps::Target::Utils::add_dir_for( cwd_exp->String(), inputs_exp, &trees); auto env_exp = eval(expr->Get("env", empty_map_exp), env); if (not env_exp->IsMap()) { throw Evaluator::EvaluationError{ fmt::format("env has to be a map of string, but found {}", env_exp->ToString())}; } for (auto const& [env_var, env_value] : env_exp->Map()) { if (not env_value->IsString()) { throw Evaluator::EvaluationError{fmt::format( "env has to be a map of string, but found {}", env_exp->ToString())}; } } auto may_fail_exp = expr->Get("may_fail", Expression::list_t{}); if (not may_fail_exp->IsList()) { throw Evaluator::EvaluationError{ fmt::format("may_fail has to be a list of " "strings, but found {}", may_fail_exp->ToString())}; } for (auto const& entry : may_fail_exp->List()) { if (not entry->IsString()) { throw Evaluator::EvaluationError{ fmt::format("may_fail has to be a list of " "strings, but found {}", may_fail_exp->ToString())}; } if (rule->Tainted().find(entry->String()) == rule->Tainted().end()) { throw Evaluator::EvaluationError{ fmt::format("may_fail contains entry {} the rule " "is not tainted with", entry->ToString())}; } } std::optional may_fail = std::nullopt; if (not may_fail_exp->List().empty()) { auto fail_msg = eval(expr->Get("fail_message", "action failed"s), env); if (not fail_msg->IsString()) { throw Evaluator::EvaluationError{ fmt::format("fail_message has to evaluate to a " "string, but got {}", fail_msg->ToString())}; } may_fail = std::optional{fail_msg->String()}; } auto no_cache_exp = expr->Get("no_cache", Expression::list_t{}); if (not no_cache_exp->IsList()) { throw Evaluator::EvaluationError{ fmt::format("no_cache has to be a list of" "strings, but found {}", no_cache_exp->ToString())}; } for (auto const& entry : no_cache_exp->List()) { if (not entry->IsString()) { throw Evaluator::EvaluationError{ fmt::format("no_cache has to be a list of" "strings, but found {}", no_cache_exp->ToString())}; } if (rule->Tainted().find(entry->String()) == rule->Tainted().end()) { throw Evaluator::EvaluationError{ fmt::format("no_cache contains entry {} the rule " "is not tainted with", entry->ToString())}; } } bool no_cache = not no_cache_exp->List().empty(); auto timeout_scale_exp = eval(expr->Get("timeout scaling", Expression::kOne), env); if (not(timeout_scale_exp->IsNumber() or timeout_scale_exp->IsNone())) { throw Evaluator::EvaluationError{ fmt::format("timeout scaling has to be number (or null " "for default), but found {}", timeout_scale_exp->ToString())}; } auto execution_properties = eval( expr->Get("execution properties", Expression::kEmptyMapExpr), env); if (execution_properties->IsNone()) { execution_properties = Expression::kEmptyMap; } if (not(execution_properties->IsMap())) { throw Evaluator::EvaluationError{ fmt::format("execution properties has to be a map of " "strings (or null for empty), but found {}", execution_properties->ToString())}; } for (auto const& [prop_name, prop_value] : execution_properties->Map()) { if (not prop_value->IsString()) { throw Evaluator::EvaluationError{fmt::format( "execution properties has to be a map of " "strings (or null for empty), but found {}", execution_properties->ToString())}; } } auto action = BuildMaps::Target::Utils::createAction( outputs_norm, output_dirs_norm, std::move(cmd), cwd_exp->String(), env_exp, may_fail, no_cache, timeout_scale_exp->IsNumber() ? timeout_scale_exp->Number() : 1.0, execution_properties, final_inputs); auto action_id = action->Id(); actions.emplace_back(std::move(action)); for (auto const& out : outputs) { result.emplace( out, ExpressionPtr{ArtifactDescription::CreateAction( action_id, ToNormalPath(out))}); } for (auto const& out : output_dirs) { result.emplace( out, ExpressionPtr{ArtifactDescription::CreateAction( action_id, ToNormalPath(out))}); } return ExpressionPtr{Expression::map_t{result}}; }}, {"BLOB", [&blobs, context](auto&& eval, auto const& expr, auto const& env) { auto data = eval(expr->Get("data", ""s), env); if (not data->IsString()) { throw Evaluator::EvaluationError{ fmt::format("BLOB data has to be a string, but got {}", data->ToString())}; } blobs.emplace_back(data->String()); return ExpressionPtr{ArtifactDescription::CreateKnown( ArtifactDigestFactory::HashDataAs( context->storage->GetHashFunction(), data->String()), ObjectType::File)}; }}, {"SYMLINK", [&blobs, context](auto&& eval, auto const& expr, auto const& env) { auto data = eval(expr->Get("data", ""s), env); if (not data->IsString()) { throw Evaluator::EvaluationError{ fmt::format("SYMLINK data has to be a string, but got {}", data->ToString())}; } if (not PathIsNonUpwards(data->String())) { throw Evaluator::EvaluationError{fmt::format( "SYMLINK data has to be non-upwards relative, but got {}", data->ToString())}; } blobs.emplace_back(data->String()); return ExpressionPtr{ArtifactDescription::CreateKnown( ArtifactDigestFactory::HashDataAs( context->storage->GetHashFunction(), data->String()), ObjectType::Symlink)}; }}, {"TREE", [&trees](auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kEmptyMapExpr), env); auto tree = ExprToTree("TREE argument", val); auto tree_id = tree->Id(); trees.emplace_back(std::move(tree)); return ExpressionPtr{ArtifactDescription::CreateTree(tree_id)}; }}, {"TREE_OVERLAY", [&tree_overlays, &trees]( auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kEmptyList), env); if (not val->IsList()) { throw Evaluator::EvaluationError{ fmt::format("TREE_OVERLAY argument has to be a list of " "stages, but found {}", val->ToString())}; } TreeOverlay::to_overlay_t parts{}; for (std::size_t i = 0; i < val->List().size(); i++) { auto& entry = val->List()[i]; auto context = fmt::format("Entry {} of TREE_OVERLAY argument", i); auto tree = ExprToTree(context, entry); auto tree_id = tree->Id(); trees.emplace_back(std::move(tree)); parts.emplace_back(ArtifactDescription::CreateTree(tree_id)); } auto overlay = std::make_shared(std::move(parts), /*disjoint=*/false); auto overlay_id = overlay->Id(); tree_overlays.emplace_back(std::move(overlay)); return ExpressionPtr{ ArtifactDescription::CreateTreeOverlay(overlay_id)}; }}, {"DISJOINT_TREE_OVERLAY", [&tree_overlays, &trees]( auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kEmptyList), env); if (not val->IsList()) { throw Evaluator::EvaluationError{fmt::format( "DISJOINT_TREE_OVERLAY argument has to be a list of " "stages, but found {}", val->ToString())}; } TreeOverlay::to_overlay_t parts{}; for (std::size_t i = 0; i < val->List().size(); i++) { auto& entry = val->List()[i]; auto context = fmt::format( "Entry {} of DISJOINT_TREE_OVERLAY argument", i); auto tree = ExprToTree(context, entry); auto tree_id = tree->Id(); trees.emplace_back(std::move(tree)); parts.emplace_back(ArtifactDescription::CreateTree(tree_id)); } auto overlay = std::make_shared(std::move(parts), /*disjoint=*/true); auto overlay_id = overlay->Id(); tree_overlays.emplace_back(std::move(overlay)); return ExpressionPtr{ ArtifactDescription::CreateTreeOverlay(overlay_id)}; }}, {"VALUE_NODE", [](auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kNone), env); if (not val->IsResult()) { throw Evaluator::EvaluationError{ "argument '$1' for VALUE_NODE not a RESULT type."}; } return ExpressionPtr{TargetNode{std::move(val)}}; }}, {"ABSTRACT_NODE", [](auto&& eval, auto const& expr, auto const& env) { auto type = eval(expr->Get("node_type", Expression::kNone), env); if (not type->IsString()) { throw Evaluator::EvaluationError{ "argument 'node_type' for ABSTRACT_NODE not a string."}; } auto string_fields = eval( expr->Get("string_fields", Expression::kEmptyMapExpr), env); if (not string_fields->IsMap()) { throw Evaluator::EvaluationError{ "argument 'string_fields' for ABSTRACT_NODE not a map."}; } auto target_fields = eval( expr->Get("target_fields", Expression::kEmptyMapExpr), env); if (not target_fields->IsMap()) { throw Evaluator::EvaluationError{ "argument 'target_fields' for ABSTRACT_NODE not a map."}; } std::optional dup_key{std::nullopt}; auto check_entries = [&dup_key](auto const& map, auto const& type_check, std::string const& fields_name, std::string const& type_name, std::optional const& disjoint_map = std::nullopt) { for (auto const& [key, list] : map->Map()) { if (not list->IsList()) { throw Evaluator::EvaluationError{fmt::format( "value for key {} in argument '{}' for " "ABSTRACT_NODE is not a list.", key, fields_name)}; } for (auto const& entry : list->List()) { if (not type_check(entry)) { throw Evaluator::EvaluationError{fmt::format( "list entry for {} in argument '{}' for " "ABSTRACT_NODE is not a {}:\n{}", key, fields_name, type_name, entry->ToString())}; } } if (disjoint_map) { if ((*disjoint_map)->Map().Find(key)) { dup_key = key; return; } } } }; auto is_string = [](auto const& e) { return e->IsString(); }; check_entries(string_fields, is_string, "string_fields", "string", target_fields); if (dup_key) { throw Evaluator::EvaluationError{ fmt::format("string_fields and target_fields are not " "disjoint maps, found duplicate key: {}.", *dup_key)}; } auto is_node = [](auto const& e) { return e->IsNode(); }; check_entries( target_fields, is_node, "target_fields", "target node"); return ExpressionPtr{TargetNode{TargetNode::Abstract{ .node_type = type->String(), .string_fields = std::move(string_fields), .target_fields = std::move(target_fields)}}}; }}, {"RESULT", [](auto&& eval, auto const& expr, auto const& env) { auto const& empty_map_exp = Expression::kEmptyMapExpr; auto artifacts = eval(expr->Get("artifacts", empty_map_exp), env); auto runfiles = eval(expr->Get("runfiles", empty_map_exp), env); auto provides = eval(expr->Get("provides", empty_map_exp), env); if (not artifacts->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "artifacts has to be a map of artifacts, but found {}", artifacts->ToString())}; } for (auto const& [path, entry] : artifacts->Map()) { if (not entry->IsArtifact()) { throw Evaluator::EvaluationError{ fmt::format("artifacts has to be a map of artifacts, " "but found {} for {}", entry->ToString(), path)}; } } auto artifacts_normal = BuildMaps::Target::Utils::artifacts_tree(artifacts); if (std::holds_alternative(artifacts_normal)) { throw Evaluator::EvaluationError{ fmt::format("artifacts conflict on path {}", std::get(artifacts_normal))}; } artifacts = std::get(artifacts_normal); auto artifacts_conflict = BuildMaps::Target::Utils::tree_conflict(artifacts); if (artifacts_conflict) { throw Evaluator::EvaluationError{ fmt::format("artifacts conflicts on subtree {}", *artifacts_conflict)}; } if (not runfiles->IsMap()) { throw Evaluator::EvaluationError{fmt::format( "runfiles has to be a map of artifacts, but found {}", runfiles->ToString())}; } for (auto const& [path, entry] : runfiles->Map()) { if (not entry->IsArtifact()) { throw Evaluator::EvaluationError{ fmt::format("runfiles has to be a map of artifacts, " "but found {} for {}", entry->ToString(), path)}; } } auto runfiles_normal = BuildMaps::Target::Utils::artifacts_tree(runfiles); if (std::holds_alternative(runfiles_normal)) { throw Evaluator::EvaluationError{ fmt::format("runfiles conflict on path {}", std::get(runfiles_normal))}; } runfiles = std::get(runfiles_normal); auto runfiles_conflict = BuildMaps::Target::Utils::tree_conflict(runfiles); if (runfiles_conflict) { throw Evaluator::EvaluationError{fmt::format( "runfiles conflicts on subtree {}", *runfiles_conflict)}; } if (not provides->IsMap()) { throw Evaluator::EvaluationError{ fmt::format("provides has to be a map, but found {}", provides->ToString())}; } return ExpressionPtr{TargetResult{.artifact_stage = artifacts, .provides = provides, .runfiles = runfiles}}; }}}); std::function annotate_object = [&deps_by_transition, &effective_conf](auto const& object) { if (not object->IsArtifact()) { // we only annotate artifacts return std::string{}; } auto occurrences = ListDependencies(object, deps_by_transition, effective_conf); if (not occurrences.empty()) { return fmt::format( "\nArtifact {} occurs in direct dependencies{}", object->ToString(), occurrences); } return fmt::format("\nArtifact {} unknown to direct dependencies", object->ToString()); }; auto result = rule->Expression()->Evaluate( expression_config, main_exp_fcts, [logger](auto const& msg) { (*logger)( fmt::format("While evaluating defining expression of rule:\n{}", msg), true); }, annotate_object); if (not result) { return; } if (not result->IsResult()) { (*logger)(fmt::format("Defining expression should evaluate to a " "RESULT, but got: {}", result->ToString()), true); return; } auto analysis_result = std::make_shared(std::move(*result).Result(), std::move(actions), std::move(blobs), std::move(trees), std::move(tree_overlays), std::move(effective_vars), std::move(tainted), std::move(implied_export), deps_info); analysis_result = result_map->Add(key.target, effective_conf, std::move(analysis_result)); (*setter)(std::move(analysis_result)); } [[nodiscard]] auto isTransition( const ExpressionPtr& ptr, std::function const& logger) -> bool { if (not ptr->IsList()) { logger(fmt::format("expected list, but got {}", ptr->ToString())); return false; } if (not std::all_of(ptr->List().begin(), ptr->List().end(), [](auto const& entry) { return entry->IsMap(); })) { logger(fmt::format("expected list of dicts, but found {}", ptr->ToString())); return false; } return true; } void withRuleDefinition( const gsl::not_null& context, const BuildMaps::Base::UserRulePtr& rule, const TargetData::Ptr& data, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null result_map) { auto param_config = key.config.Prune(data->target_vars); // Evaluate the config_fields std::unordered_map params; params.reserve(rule->ConfigFields().size() + rule->TargetFields().size() + rule->ImplicitTargetExps().size()); for (auto const& field_name : rule->ConfigFields()) { auto const& field_expression = data->config_exprs[field_name]; auto field_value = field_expression.Evaluate( param_config, {}, [&logger, &field_name](auto const& msg) { (*logger)(fmt::format("While evaluating config field {}:\n{}", field_name, msg), true); }); if (not field_value) { return; } if (not field_value->IsList()) { (*logger)(fmt::format("Config field {} should evaluate to a list " "of strings, but got{}", field_name, field_value->ToString()), true); return; } for (auto const& entry : field_value->List()) { if (not entry->IsString()) { (*logger)(fmt::format("Config field {} should evaluate to a " "list of strings, but got{}", field_name, field_value->ToString()), true); return; } } params.emplace(field_name, field_value); } // Evaluate config transitions auto config_trans_fcts = FunctionMap::MakePtr( "FIELD", [¶ms](auto&& eval, auto const& expr, auto const& env) { auto name = eval(expr["name"], env); if (not name->IsString()) { throw Evaluator::EvaluationError{ fmt::format("FIELD argument 'name' should evaluate to a " "string, but got {}", name->ToString())}; } auto it = params.find(name->String()); if (it == params.end()) { throw Evaluator::EvaluationError{ fmt::format("FIELD {} unknown", name->String())}; } return it->second; }); auto const& config_vars = rule->ConfigVars(); auto expression_config = key.config.Prune(config_vars); std::unordered_map config_transitions; config_transitions.reserve(rule->TargetFields().size() + rule->ImplicitTargets().size() + rule->AnonymousDefinitions().size()); for (auto const& target_field_name : rule->TargetFields()) { auto exp = rule->ConfigTransitions().at(target_field_name); auto transition_logger = [&logger, &target_field_name](auto const& msg) { (*logger)( fmt::format("While evaluating config transition for {}:\n{}", target_field_name, msg), true); }; auto transition = exp->Evaluate( expression_config, config_trans_fcts, transition_logger); if (not transition) { return; } if (not isTransition(transition, transition_logger)) { return; } config_transitions.emplace(target_field_name, transition); } for (const auto& name_value : rule->ImplicitTargets()) { auto implicit_field_name = name_value.first; auto exp = rule->ConfigTransitions().at(implicit_field_name); auto transition_logger = [&logger, &implicit_field_name](auto const& msg) { (*logger)(fmt::format("While evaluating config transition for " "implicit {}:\n{}", implicit_field_name, msg), true); }; auto transition = exp->Evaluate( expression_config, config_trans_fcts, transition_logger); if (not transition) { return; } if (not isTransition(transition, transition_logger)) { return; } config_transitions.emplace(implicit_field_name, transition); } for (const auto& entry : rule->AnonymousDefinitions()) { auto const& anon_field_name = entry.first; auto exp = rule->ConfigTransitions().at(anon_field_name); auto transition_logger = [&logger, &anon_field_name](auto const& msg) { (*logger)(fmt::format("While evaluating config transition for " "anonymous {}:\n{}", anon_field_name, msg), true); }; auto transition = exp->Evaluate( expression_config, config_trans_fcts, transition_logger); if (not transition) { return; } if (not isTransition(transition, transition_logger)) { return; } config_transitions.emplace(anon_field_name, transition); } // Request dependencies std::unordered_map> anon_positions; anon_positions.reserve(rule->AnonymousDefinitions().size()); for (auto const& [_, def] : rule->AnonymousDefinitions()) { anon_positions.emplace(def.target, std::vector{}); } std::vector dependency_keys; std::vector transition_keys; for (auto const& target_field_name : rule->TargetFields()) { auto const& deps_expression = data->target_exprs[target_field_name]; auto deps_names = deps_expression.Evaluate( param_config, {}, [&logger, &target_field_name](auto const& msg) { (*logger)( fmt::format("While evaluating target parameter {}:\n{}", target_field_name, msg), true); }); if (not deps_names) { return; } if (not deps_names->IsList()) { (*logger)(fmt::format("Target parameter {} should evaluate to a " "list, but got {}", target_field_name, deps_names->ToString()), true); return; } Expression::list_t dep_target_exps; if (data->parse_target_names) { dep_target_exps.reserve(deps_names->List().size()); for (const auto& dep_name : deps_names->List()) { auto target = BuildMaps::Base::ParseEntityNameFromExpression( dep_name, key.target, context->repo_config, [&logger, &target_field_name, &dep_name]( std::string const& parse_err) { (*logger)(fmt::format("Parsing entry {} in target " "field {} failed with:\n{}", dep_name->ToString(), target_field_name, parse_err), true); }); if (not target) { return; } dep_target_exps.emplace_back(*target); } } else { dep_target_exps = deps_names->List(); } auto anon_pos = anon_positions.find(target_field_name); auto const& transitions = config_transitions[target_field_name]->List(); for (const auto& transition : transitions) { auto transitioned_config = key.config.Update(transition); for (const auto& dep : dep_target_exps) { if (anon_pos != anon_positions.end()) { anon_pos->second.emplace_back(dependency_keys.size()); } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = dep->Name(), .config = transitioned_config}); transition_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = dep->Name(), .config = Configuration{transition}}); } } params.emplace(target_field_name, ExpressionPtr{std::move(dep_target_exps)}); } auto declared_count = dependency_keys.size(); for (auto const& [implicit_field_name, implicit_target] : rule->ImplicitTargets()) { auto anon_pos = anon_positions.find(implicit_field_name); auto transitions = config_transitions[implicit_field_name]->List(); for (const auto& transition : transitions) { auto transitioned_config = key.config.Update(transition); for (const auto& dep : implicit_target) { if (anon_pos != anon_positions.end()) { anon_pos->second.emplace_back(dependency_keys.size()); } dependency_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = dep, .config = transitioned_config}); transition_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = dep, .config = Configuration{transition}}); } } } params.insert(rule->ImplicitTargetExps().begin(), rule->ImplicitTargetExps().end()); auto declared_and_implicit_count = dependency_keys.size(); (*subcaller)( dependency_keys, [context, transition_keys = std::move(transition_keys), declared_count, declared_and_implicit_count, rule, data, key, params = std::move(params), setter, logger, result_map, subcaller, config_transitions = std::move(config_transitions), anon_positions = std::move(anon_positions)](auto const& values) mutable { // Now that all non-anonymous targets have been evaluated we can // read their provides map to construct and evaluate anonymous // targets. std::vector anonymous_keys; for (auto const& [name, def] : rule->AnonymousDefinitions()) { Expression::list_t anon_names{}; for (auto pos : anon_positions.at(def.target)) { auto const& provider_value = (*values[pos])->Provides()->Map().Find(def.provider); if (not provider_value) { (*logger)( fmt::format("Provider {} in {} does not exist", def.provider, def.target), true); return; } auto const& exprs = **provider_value; if (not exprs->IsList()) { (*logger)(fmt::format("Provider {} in {} must be list " "of target nodes but found: {}", def.provider, def.target, exprs->ToString()), true); return; } auto const& list = exprs->List(); anon_names.reserve(anon_names.size() + list.size()); for (auto const& node : list) { if (not node->IsNode()) { (*logger)( fmt::format("Entry in provider {} in {} must " "be target node but found: {}", def.provider, def.target, node->ToString()), true); return; } anon_names.emplace_back(BuildMaps::Base::EntityName{ BuildMaps::Base::AnonymousTarget{ .rule_map = def.rule_map, .target_node = node}}); } } for (const auto& transition : config_transitions.at(name)->List()) { auto transitioned_config = key.config.Update(transition); for (auto const& anon : anon_names) { anonymous_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = anon->Name(), .config = transitioned_config}); transition_keys.emplace_back( BuildMaps::Target::ConfiguredTarget{ .target = anon->Name(), .config = Configuration{transition}}); } } params.emplace(name, ExpressionPtr{std::move(anon_names)}); } (*subcaller)( anonymous_keys, [context, dependency_values = values, transition_keys = std::move(transition_keys), declared_count, declared_and_implicit_count, rule, data, key, params = std::move(params), setter, logger, result_map](auto const& values) mutable { // Join dependency values and anonymous values dependency_values.insert( dependency_values.end(), values.begin(), values.end()); withDependencies(context, transition_keys, dependency_values, declared_count, declared_and_implicit_count, rule, data, key, params, setter, logger, result_map); }, logger); }, logger); } void withTargetsFile( const gsl::not_null& context, const BuildMaps::Target::ConfiguredTarget& key, const nlohmann::json& targets_file, const gsl::not_null& source_target, const gsl::not_null& rule_map, const gsl::not_null& ts, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null result_map) { auto desc_it = targets_file.find(key.target.GetNamedTarget().name); if (desc_it == targets_file.end()) { // Not a defined taraget, treat as source target source_target->ConsumeAfterKeysReady( ts, {key.target}, [setter](auto values) { (*setter)(AnalysedTargetPtr{*values[0]}); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While analysing target {} as implicit " "source target:\n{}", target.ToString(), msg), fatal); }); } else { nlohmann::json desc = *desc_it; auto rule_it = desc.find("type"); if (rule_it == desc.end()) { (*logger)( fmt::format("No type specified in the definition of target {}", key.target.ToString()), true); return; } // Handle built-in rule, if it is auto handled_as_builtin = BuildMaps::Target::HandleBuiltin(context, *rule_it, desc, key, subcaller, setter, logger, result_map); if (handled_as_builtin) { return; } // Not a built-in rule, so has to be a user rule auto rule_name = BuildMaps::Base::ParseEntityNameFromJson( *rule_it, key.target, context->repo_config, [&logger, &rule_it, &key](std::string const& parse_err) { (*logger)(fmt::format("Parsing rule name {} for target {} " "failed with:\n{}", rule_it->dump(), key.target.ToString(), parse_err), true); }); if (not rule_name) { return; } auto desc_reader = BuildMaps::Base::FieldReader::CreatePtr( desc, key.target, fmt::format("{} target", rule_name->ToString()), logger); if (not desc_reader) { return; } rule_map->ConsumeAfterKeysReady( ts, {*rule_name}, [desc = std::move(desc_reader), subcaller, setter, logger, key, context, result_map, rn = *rule_name](auto values) { auto data = TargetData::FromFieldReader(*values[0], desc); if (not data) { (*logger)(fmt::format("Failed to read data from target {} " "with rule {}", key.target.ToString(), rn.ToString()), /*fatal=*/true); return; } withRuleDefinition( context, *values[0], data, key, subcaller, setter, std::make_shared( [logger, key, rn](auto const& msg, auto fatal) { (*logger)( fmt::format( "While analysing {} target {}:\n{}", rn.ToString(), key.ToShortString( Evaluator::GetExpressionLogLimit()), msg), fatal); }), result_map); }, [logger, rule = *rule_name, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While looking up rule {} for {}:\n{}", rule.ToString(), target.ToString(), msg), fatal); }); } } void withTargetNode( const gsl::not_null& context, const BuildMaps::Target::ConfiguredTarget& key, const gsl::not_null& rule_map, const gsl::not_null& ts, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null result_map) { auto const& target_node = key.target.GetAnonymousTarget().target_node->Node(); auto const& rule_mapping = key.target.GetAnonymousTarget().rule_map->Map(); if (target_node.IsValue()) { // fixed value node, create analysed target from result auto const& val = target_node.GetValue(); (*setter)(std::make_shared( AnalysedTarget{val->Result(), {}, {}, {}, {}, {}, {}, {}, TargetGraphInformation::kSource})); } else { // abstract target node, lookup rule and instantiate target auto const& abs = target_node.GetAbstract(); auto rule_name = rule_mapping.Find(abs.node_type); if (not rule_name) { (*logger)(fmt::format( "Cannot resolve type of node {} via rule map " "{}", target_node.ToString(), key.target.GetAnonymousTarget().rule_map->ToString()), /*fatal=*/true); return; } rule_map->ConsumeAfterKeysReady( ts, {(**rule_name)->Name()}, [abs, subcaller, setter, logger, key, context, result_map, rn = **rule_name](auto values) { auto data = TargetData::FromTargetNode( *values[0], abs, key.target.GetAnonymousTarget().rule_map, logger); if (not data) { (*logger)(fmt::format("Failed to read data from target {} " "with rule {}", key.target.ToString(), rn->ToString()), /*fatal=*/true); return; } withRuleDefinition(context, *values[0], data, key, subcaller, setter, std::make_shared( [logger, target = key.target, rn]( auto const& msg, auto fatal) { (*logger)( fmt::format("While analysing {} " "target {}:\n{}", rn->ToString(), target.ToString(), msg), fatal); }), result_map); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While looking up rule for {}:\n{}", target.ToString(), msg), fatal); }); } } void TreeTarget( const gsl::not_null& context, const BuildMaps::Target::ConfiguredTarget& key, const gsl::not_null& ts, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& result_map, const gsl::not_null& directory_entries) { const auto& target = key.target.GetNamedTarget(); const auto dir_name = std::filesystem::path{target.module} / target.name; auto target_module = BuildMaps::Base::ModuleName{target.repository, dir_name}; directory_entries->ConsumeAfterKeysReady( ts, {target_module}, [context, setter, subcaller, target, key, result_map, logger, dir_name]( auto values) { // expected values.size() == 1 const auto& dir_entries = *values[0]; auto known_tree = dir_entries.AsKnownTree( context->storage->GetHashFunction().GetType(), target.repository); if (known_tree) { auto tree = ExpressionPtr{ Expression::map_t{target.name, ExpressionPtr{*known_tree}}}; auto analysis_result = std::make_shared( TargetResult{.artifact_stage = tree, .provides = {}, .runfiles = tree}, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{}, std::set{}, std::set{}, TargetGraphInformation::kSource); analysis_result = result_map->Add(key.target, {}, std::move(analysis_result)); (*setter)(std::move(analysis_result)); return; } Logger::Log(LogLevel::Debug, [&key]() { return fmt::format( "Source tree reference for non-known tree {}", key.target.ToString()); }); context->statistics->IncrementTreesAnalysedCounter(); using BuildMaps::Target::ConfiguredTarget; std::vector v; for (const auto& x : dir_entries.FilesIterator()) { v.emplace_back(ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ target.repository, dir_name, x, BuildMaps::Base::ReferenceType::kFile}, .config = Configuration{}}); } for (const auto& x : dir_entries.SymlinksIterator()) { v.emplace_back(ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ target.repository, dir_name, x, BuildMaps::Base::ReferenceType::kSymlink}, .config = Configuration{}}); } for (const auto& x : dir_entries.DirectoriesIterator()) { v.emplace_back(ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ target.repository, dir_name, x, BuildMaps::Base::ReferenceType::kTree}, .config = Configuration{}}); } (*subcaller)( std::move(v), [setter, key, result_map, name = target.name]( const auto& values) mutable { std::unordered_map artifacts; artifacts.reserve(values.size()); for (const auto& x : values) { auto val = x->get()->RunFiles(); auto const& [input_path, artifact] = *(val->Map().begin()); auto norm_path = ToNormalPath(std::filesystem::path{input_path}) .string(); artifacts.emplace(std::move(norm_path), artifact->Artifact()); } auto tree = std::make_shared(std::move(artifacts)); auto tree_id = tree->Id(); auto tree_map = ExpressionPtr{Expression::map_t{ name, ExpressionPtr{ ArtifactDescription::CreateTree(tree_id)}}}; auto analysis_result = std::make_shared( TargetResult{.artifact_stage = tree_map, .provides = {}, .runfiles = tree_map}, std::vector{}, std::vector{}, std::vector{tree}, std::vector{}, std::unordered_set{}, std::set{}, std::set{}, TargetGraphInformation::kSource); analysis_result = result_map->Add( key.target, {}, std::move(analysis_result)); (*setter)(std::move(analysis_result)); }, logger); }, [logger, target = key.target](auto const& msg, bool fatal) { (*logger)(fmt::format("While analysing entries of {}: {}", target.ToString(), msg), fatal); }); } void GlobResult(const std::vector& values, const BuildMaps::Target::TargetMap::SetterPtr& setter) { auto result = Expression::map_t::underlying_map_t{}; for (auto const& value : values) { for (auto const& [k, v] : (*value)->Artifacts()->Map()) { result[k] = v; } } auto stage = ExpressionPtr{Expression::map_t{result}}; auto target = std::make_shared( TargetResult{.artifact_stage = stage, .provides = Expression::kEmptyMap, .runfiles = stage}, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{}, std::set{}, std::set{}, TargetGraphInformation::kSource); (*setter)(std::move(target)); } void GlobTargetWithDirEntry( const BuildMaps::Base::EntityName& key, const gsl::not_null& ts, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const gsl::not_null& source_target_map, const FileRoot::DirectoryEntries& dir) { auto const& target = key.GetNamedTarget(); auto const& pattern = target.name; std::vector matches; for (auto const& x : dir.FilesIterator()) { if (fnmatch(pattern.c_str(), x.c_str(), 0) == 0) { matches.emplace_back(target.repository, target.module, x, BuildMaps::Base::ReferenceType::kFile); } } for (auto const& x : dir.SymlinksIterator()) { if (fnmatch(pattern.c_str(), x.c_str(), 0) == 0) { matches.emplace_back(target.repository, target.module, x, BuildMaps::Base::ReferenceType::kSymlink); } } source_target_map->ConsumeAfterKeysReady( ts, matches, [setter](auto values) { GlobResult(values, setter); }, [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While handling matching file targets:\n{}", msg), fatal); }); } } // namespace namespace BuildMaps::Target { auto CreateTargetMap( const gsl::not_null& context, const gsl::not_null& source_target_map, const gsl::not_null& targets_file_map, const gsl::not_null& rule_map, const gsl::not_null& directory_entries_map, const gsl::not_null& absent_target_map, const gsl::not_null& result_map, std::size_t jobs) -> TargetMap { auto target_reader = [source_target_map, targets_file_map, rule_map, directory_entries_map, absent_target_map, result_map, context](auto ts, auto setter, auto logger, auto subcaller, auto key) { if (key.target.IsAnonymousTarget()) { withTargetNode(context, key, rule_map, ts, subcaller, setter, logger, result_map); } else if (key.target.GetNamedTarget().reference_t == BuildMaps::Base::ReferenceType::kTree) { auto wrapped_logger = std::make_shared( [logger, target = key.target](auto const& msg, bool fatal) { (*logger)(fmt::format("While analysing {} as explicit " "tree reference:\n{}", target.ToString(), msg), fatal); }); TreeTarget(context, key, ts, subcaller, setter, wrapped_logger, result_map, directory_entries_map); } else if (key.target.GetNamedTarget().reference_t == BuildMaps::Base::ReferenceType::kFile) { // Not a defined target, treat as source target source_target_map->ConsumeAfterKeysReady( ts, {key.target}, [setter](auto values) { (*setter)(AnalysedTargetPtr{*values[0]}); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While analysing target {} as " "explicit source target:\n{}", target.ToString(), msg), fatal); }); } else if (key.target.GetNamedTarget().reference_t == BuildMaps::Base::ReferenceType::kSymlink) { // Not a defined target, treat as source target source_target_map->ConsumeAfterKeysReady( ts, {key.target}, [setter](auto values) { (*setter)(AnalysedTargetPtr{*values[0]}); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While analysing target {} as " "symlink:\n{}", target.ToString(), msg), fatal); }); } else if (key.target.GetNamedTarget().reference_t == BuildMaps::Base::ReferenceType::kGlob) { auto wrapped_logger = std::make_shared( [logger, target = key.target](auto const& msg, bool fatal) { (*logger)(fmt::format("While analysing {} as glob:\n{}", target.ToString(), msg), fatal); }); auto const& target = key.target; directory_entries_map->ConsumeAfterKeysReady( ts, {target.ToModule()}, [target, ts, setter, wrapped_logger, source_target_map]( auto values) { GlobTargetWithDirEntry(target, ts, setter, wrapped_logger, source_target_map, *values[0]); }, [target, logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While reading directory for {}:\n{}", target.ToString(), msg), fatal); } ); } #ifndef BOOTSTRAP_BUILD_TOOL else if (auto const* const file_root = context->repo_config->TargetRoot( key.target.ToModule().repository); file_root != nullptr and file_root->IsAbsent()) { if (context->serve == nullptr) { (*logger)( fmt::format("Root for target {} is absent, but no serve " "endpoint was configured. Please provide " "--remote-serve-address and retry.", key.target.ToJson().dump()), /*is_fatal=*/true); return; } if (not context->serve->CheckServeRemoteExecution()) { (*logger)( "Inconsistent remote execution endpoint and serve endpoint" "configuration detected.", /*is_fatal=*/true); return; } absent_target_map->ConsumeAfterKeysReady( ts, {key}, [setter](auto values) { (*setter)(AnalysedTargetPtr{*values[0]}); }, [logger, key](auto msg, auto fatal) { (*logger)( fmt::format("While processing absent target {}:\n{}", key.ToShortString( Evaluator::GetExpressionLogLimit()), msg), fatal); }); } #endif else { targets_file_map->ConsumeAfterKeysReady( ts, {key.target.ToModule()}, [key, context, source_target_map, rule_map, ts, subcaller = std::move(subcaller), setter = std::move(setter), logger, result_map](auto values) { withTargetsFile(context, key, *values[0], source_target_map, rule_map, ts, subcaller, setter, logger, result_map); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While searching targets " "description for {}:\n{}", target.ToString(), msg), fatal); }); } }; return AsyncMapConsumer(target_reader, jobs); } } // namespace BuildMaps::Target just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/target_map.hpp000066400000000000000000000046671516554100600320230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_MAP_HPP #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include "src/buildtool/build_engine/base_maps/source_map.hpp" #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" namespace BuildMaps::Target { using TargetMap = AsyncMapConsumer; auto CreateTargetMap( const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, const gsl::not_null&, std::size_t jobs = 0) -> TargetMap; // use explicit cast to std::function to allow template deduction when used static const std::function kConfiguredTargetPrinter = [](ConfiguredTarget const& x) -> std::string { return x.ToString(); }; auto IsBuiltInRule(nlohmann::json const& rule_type) -> bool; } // namespace BuildMaps::Target #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/utils.cpp000066400000000000000000000276361516554100600310340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/utils.hpp" #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/path_hash.hpp" auto BuildMaps::Target::Utils::obtainTargetByName( const SubExprEvaluator& eval, const ExpressionPtr& expr, const Configuration& env, const Base::EntityName& current, const gsl::not_null& repo_config, std::unordered_map const& deps_by_transition) -> AnalysedTargetPtr { auto const& empty_map_exp = Expression::kEmptyMapExpr; auto reference = eval(expr["dep"], env); std::string error{}; auto target = BuildMaps::Base::ParseEntityNameFromExpression( reference, current, repo_config, [&error](std::string const& parse_err) { error = parse_err; }); if (not target) { throw Evaluator::EvaluationError{ fmt::format("Parsing target name {} failed with:\n{}", reference->ToString(), error)}; } auto transition = eval(expr->Get("transition", empty_map_exp), env); auto it = deps_by_transition.find(BuildMaps::Target::ConfiguredTarget{ .target = *target, .config = Configuration{transition}}); if (it == deps_by_transition.end()) { throw Evaluator::EvaluationError{fmt::format( "Reference to undeclared dependency {} in transition {}", reference->ToString(), transition->ToString())}; } return it->second; } auto BuildMaps::Target::Utils::obtainTarget( const SubExprEvaluator& eval, const ExpressionPtr& expr, const Configuration& env, std::unordered_map const& deps_by_transition) -> AnalysedTargetPtr { auto const& empty_map_exp = Expression::kEmptyMapExpr; auto reference = eval(expr["dep"], env); if (not reference->IsName()) { throw Evaluator::EvaluationError{ fmt::format("Not a target name: {}", reference->ToString())}; } auto transition = eval(expr->Get("transition", empty_map_exp), env); auto it = deps_by_transition.find(BuildMaps::Target::ConfiguredTarget{ .target = reference->Name(), .config = Configuration{transition}}); if (it == deps_by_transition.end()) { throw Evaluator::EvaluationError{fmt::format( "Reference to undeclared dependency {} in transition {}", reference->ToString(), transition->ToString())}; } return it->second; } auto BuildMaps::Target::Utils::keys_expr(const ExpressionPtr& map) -> ExpressionPtr { auto const& m = map->Map(); auto result = Expression::list_t{}; result.reserve(m.size()); std::for_each(m.begin(), m.end(), [&](auto const& item) { result.emplace_back(ExpressionPtr{item.first}); }); return ExpressionPtr{result}; } auto BuildMaps::Target::Utils::artifacts_tree(const ExpressionPtr& map) -> std::variant { auto result = Expression::map_t::underlying_map_t{}; for (auto const& [key, artifact] : map->Map()) { auto location = ToNormalPath(std::filesystem::path{key}).string(); if (auto it = result.find(location); it != result.end() and not(it->second == artifact)) { return location; } result.emplace(std::move(location), artifact); } return ExpressionPtr{Expression::map_t{result}}; } auto BuildMaps::Target::Utils::tree_conflict(const ExpressionPtr& map) -> std::optional { // Work around the fact that std::hash is missing // in some libraries struct PathHash { auto operator()(std::filesystem::path const& p) const noexcept -> std::size_t { return std::hash{}(p); } }; std::unordered_set blocked{}; blocked.reserve(map->Map().size()); for (auto const& [path, artifact] : map->Map()) { if (path == "." and map->Map().size() > 1) { return "."; } auto p = std::filesystem::path{path}; if (p.is_absolute()) { return p.string(); } if (*p.begin() == "..") { return p.string(); } auto insert_result = blocked.insert(p); if (not insert_result.second) { return p.string(); // duplicate path } for (p = p.parent_path(); not p.empty(); p = p.parent_path()) { if (blocked.contains(p)) { // Another artifact at a parent path position return p.string(); } } } return std::nullopt; } auto BuildMaps::Target::Utils::add_dir_for( const std::string& cwd, ExpressionPtr stage, gsl::not_null*> trees) -> ExpressionPtr { // if working top-level, there is nothing to add; this is also // the common case if ((cwd.empty()) or (cwd == ".")) { return stage; } auto cwd_path = std::filesystem::path{cwd}; for (auto const& [path, artifact] : stage->Map()) { if ((path.empty()) or (path == ".")) { // top-level artifact (tree); cannot add tree for cwd return stage; } auto p = std::filesystem::path{path}; for (auto c = cwd_path; not c.empty(); c = c.parent_path()) { if (c == p) { // adding cwd would add a tree conflict; so nothing to add return stage; } } for (; not p.empty(); p = p.parent_path()) { if (p == cwd_path) { // adding cwd would add a tree conflict; so nothing to add return stage; } } } // As we can add cwd without tree conflicts, we have to in order to // ensure that installing this stage implies a directory at cwd. std::unordered_map artifacts{}; auto empty_tree = std::make_shared(std::move(artifacts)); auto empty_tree_id = empty_tree->Id(); trees->emplace_back(std::move(empty_tree)); auto empty_tree_exp = ExpressionPtr{ArtifactDescription::CreateTree(empty_tree_id)}; auto cwd_tree = ExpressionPtr{Expression::map_t{cwd, empty_tree_exp}}; return ExpressionPtr{Expression::map_t{stage, cwd_tree}}; } auto BuildMaps::Target::Utils::getTainted( std::set* tainted, const Configuration& config, const ExpressionPtr& tainted_exp, const BuildMaps::Target::TargetMap::LoggerPtr& logger) -> bool { if (not tainted_exp) { return false; } auto tainted_val = tainted_exp.Evaluate(config, {}, [logger](auto const& msg) { (*logger)(fmt::format("While evaluating tainted:\n{}", msg), true); }); if (not tainted_val) { return false; } if (not tainted_val->IsList()) { (*logger)(fmt::format("tainted should evaluate to a list of strings, " "but got {}", tainted_val->ToString()), true); return false; } for (auto const& entry : tainted_val->List()) { if (not entry->IsString()) { (*logger)(fmt::format("tainted should evaluate to a list of " "strings, but got {}", tainted_val->ToString()), true); return false; } tainted->insert(entry->String()); } return true; } namespace { auto hash_vector(HashFunction hash_function, std::vector const& vec) -> std::string { auto hasher = hash_function.MakeHasher(); for (auto const& s : vec) { hasher.Update(hash_function.PlainHashData(s).Bytes()); } return std::move(hasher).Finalize().Bytes(); } } // namespace auto BuildMaps::Target::Utils::createAction( const ActionDescription::outputs_t& output_files, const ActionDescription::outputs_t& output_dirs, std::vector command, std::string cwd, const ExpressionPtr& env, std::optional may_fail, bool no_cache, double timeout_scale, const ExpressionPtr& execution_properties_exp, const ExpressionPtr& inputs_exp) -> ActionDescription::Ptr { // The type of HashFunction is irrelevant here. It is used for // identification and quick comparison of descriptions. SHA256 is used. HashFunction hash_function{HashFunction::Type::PlainSHA256}; auto hasher = hash_function.MakeHasher(); hasher.Update("ACTION:"); hasher.Update(hash_vector(hash_function, output_files)); hasher.Update(hash_vector(hash_function, output_dirs)); hasher.Update(hash_vector(hash_function, command)); hasher.Update(hash_vector(hash_function, std::vector{cwd})); hasher.Update(env->ToHash()); hasher.Update(hash_vector(hash_function, may_fail ? std::vector{*may_fail} : std::vector{})); hasher.Update(no_cache ? std::string{"N"} : std::string{"Y"}); hasher.Update(fmt::format("{:+24a}", timeout_scale)); hasher.Update(execution_properties_exp->ToHash()); hasher.Update(inputs_exp->ToHash()); auto action_id = std::move(hasher).Finalize().HexString(); std::map env_vars{}; for (auto const& [env_var, env_value] : env->Map()) { env_vars.emplace(env_var, env_value->String()); } std::map execution_properties{}; for (auto const& [prop_name, prop_value] : execution_properties_exp->Map()) { execution_properties.emplace(prop_name, prop_value->String()); } ActionDescription::inputs_t inputs; inputs.reserve(inputs_exp->Map().size()); for (auto const& [input_path, artifact] : inputs_exp->Map()) { inputs.emplace(input_path, artifact->Artifact()); } return std::make_shared(output_files, output_dirs, Action{std::move(action_id), std::move(command), std::move(cwd), std::move(env_vars), std::move(may_fail), no_cache, timeout_scale, execution_properties}, std::move(inputs)); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/build_engine/target_map/utils.hpp000066400000000000000000000067441516554100600310360ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_UTILS_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/tree.hpp" namespace BuildMaps::Target::Utils { auto obtainTargetByName(const SubExprEvaluator&, const ExpressionPtr&, const Configuration&, const Base::EntityName&, const gsl::not_null&, std::unordered_map const&) -> AnalysedTargetPtr; auto obtainTarget(const SubExprEvaluator&, const ExpressionPtr&, const Configuration&, std::unordered_map const&) -> AnalysedTargetPtr; auto keys_expr(const ExpressionPtr& map) -> ExpressionPtr; auto artifacts_tree(const ExpressionPtr& map) -> std::variant; auto tree_conflict(const ExpressionPtr& /* map */) -> std::optional; auto add_dir_for(const std::string& cwd, ExpressionPtr stage, gsl::not_null*> trees) -> ExpressionPtr; auto getTainted(std::set* tainted, const Configuration& config, const ExpressionPtr& tainted_exp, const BuildMaps::Target::TargetMap::LoggerPtr& logger) -> bool; auto createAction(const ActionDescription::outputs_t& output_files, const ActionDescription::outputs_t& output_dirs, std::vector command, std::string cwd, const ExpressionPtr& env, std::optional may_fail, bool no_cache, double timeout_scale, const ExpressionPtr& execution_properties_exp, const ExpressionPtr& inputs_exp) -> ActionDescription::Ptr; } // namespace BuildMaps::Target::Utils #endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/000077500000000000000000000000001516554100600236735ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/TARGETS000066400000000000000000000167541516554100600247440ustar00rootroot00000000000000{ "clidefaults": { "type": ["@", "rules", "CC", "library"] , "name": ["clidefaults"] , "hdrs": ["clidefaults.hpp"] , "stage": ["src", "buildtool", "common"] , "deps": [["src/buildtool/logging", "log_level"]] } , "retry_cli": { "type": ["@", "rules", "CC", "library"] , "name": ["retry_cli"] , "hdrs": ["retry_cli.hpp"] , "stage": ["src", "buildtool", "common"] , "deps": [["@", "cli11", "", "cli11"], ["@", "gsl", "", "gsl"]] } , "cli": { "type": ["@", "rules", "CC", "library"] , "name": ["cli"] , "hdrs": ["cli.hpp"] , "deps": [ "clidefaults" , ["@", "cli11", "", "cli11"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system/symlinks", "resolve_special"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "build_utils"] , ["src/utils/cpp", "path"] ] , "stage": ["src", "buildtool", "common"] } , "bazel_types": { "type": ["@", "rules", "CC", "library"] , "name": ["bazel_types"] , "hdrs": ["bazel_types.hpp"] , "proto": [["@", "bazel_remote_apis", "", "remote_execution_proto"]] , "stage": ["src", "buildtool", "common"] , "deps": [["@", "protoc", "", "libprotobuf"]] } , "bazel_digest_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["bazel_digest_factory"] , "hdrs": ["bazel_digest_factory.hpp"] , "srcs": ["bazel_digest_factory.cpp"] , "deps": [ "bazel_types" , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ "protocol_traits" , ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hasher"] ] , "stage": ["src", "buildtool", "common"] } , "artifact_blob": { "type": ["@", "rules", "CC", "library"] , "name": ["artifact_blob"] , "hdrs": ["artifact_blob.hpp"] , "srcs": ["artifact_blob.cpp"] , "deps": [ "common" , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] , ["src/utils/cpp", "tmp_dir"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "in_place_visitor"] ] , "stage": ["src", "buildtool", "common"] } , "common": { "type": ["@", "rules", "CC", "library"] , "name": ["common"] , "hdrs": [ "action.hpp" , "artifact_digest.hpp" , "artifact_digest_factory.hpp" , "artifact.hpp" , "identifier.hpp" ] , "srcs": ["artifact_digest_factory.cpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hash_combine"] ] , "private-deps": [ "bazel_digest_factory" , "bazel_types" , "protocol_traits" , ["@", "gsl", "", "gsl"] ] , "stage": ["src", "buildtool", "common"] } , "artifact_description": { "type": ["@", "rules", "CC", "library"] , "name": ["artifact_description"] , "hdrs": ["artifact_description.hpp"] , "srcs": ["artifact_description.cpp"] , "deps": [ "common" , ["@", "json", "", "json"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "object_type"] ] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "json"] ] , "stage": ["src", "buildtool", "common"] } , "action_description": { "type": ["@", "rules", "CC", "library"] , "name": ["action_description"] , "hdrs": ["action_description.hpp"] , "deps": [ "artifact_description" , "common" , ["@", "json", "", "json"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "json"] ] , "stage": ["src", "buildtool", "common"] } , "tree": { "type": ["@", "rules", "CC", "library"] , "name": ["tree"] , "hdrs": ["tree.hpp"] , "deps": [ "action_description" , "artifact_description" , "common" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/crypto", "hash_function"] ] , "stage": ["src", "buildtool", "common"] } , "tree_overlay": { "type": ["@", "rules", "CC", "library"] , "name": ["tree_overlay"] , "hdrs": ["tree_overlay.hpp"] , "deps": [ "action_description" , "artifact_description" , "common" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/crypto", "hash_function"] ] , "stage": ["src", "buildtool", "common"] } , "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "hdrs": ["repository_config.hpp"] , "srcs": ["repository_config.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_tree"] , ["src/buildtool/file_system", "precomputed_root"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "atomic_value"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "common"] , "private-deps": [ ["src/buildtool/file_system", "git_tree_utils"] , ["src/utils/automata", "dfa_minimizer"] ] } , "user_structs": { "type": ["@", "rules", "CC", "library"] , "name": ["user_structs"] , "hdrs": ["user_structs.hpp"] , "stage": ["src", "buildtool", "common"] , "deps": [ ["@", "json", "", "json"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/main", "constants"] ] } , "location": { "type": ["@", "rules", "CC", "library"] , "name": ["location"] , "hdrs": ["location.hpp"] , "srcs": ["location.cpp"] , "deps": [["@", "json", "", "json"], ["src/utils/cpp", "expected"]] , "stage": ["src", "buildtool", "common"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "git_hashes_converter": { "type": ["@", "rules", "CC", "library"] , "name": ["git_hashes_converter"] , "hdrs": ["git_hashes_converter.hpp"] , "deps": [ ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "common"] } , "protocol_traits": { "type": ["@", "rules", "CC", "library"] , "name": ["protocol_traits"] , "hdrs": ["protocol_traits.hpp"] , "deps": [["src/buildtool/crypto", "hash_function"]] , "stage": ["src", "buildtool", "common"] } , "statistics": { "type": ["@", "rules", "CC", "library"] , "name": ["statistics"] , "hdrs": ["statistics.hpp"] , "stage": ["src", "buildtool", "common"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/action.hpp000066400000000000000000000112531516554100600256630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP #include #include #include #include #include #include "src/buildtool/common/identifier.hpp" class Action { public: using LocalPath = std::string; Action(std::string action_id, std::vector command, std::string cwd, std::map env_vars, std::optional may_fail, bool no_cache, double timeout_scale, std::map execution_properties) : id_{std::move(action_id)}, command_{std::move(command)}, cwd_{std::move(cwd)}, env_{std::move(env_vars)}, may_fail_{std::move(may_fail)}, no_cache_{no_cache}, timeout_scale_{timeout_scale}, execution_properties_{std::move(std::move(execution_properties))} {} Action(std::string action_id, std::vector command, std::map env_vars) : Action(std::move(action_id), std::move(command), "", std::move(env_vars), std::nullopt, false, 1.0, std::map{}) {} [[nodiscard]] auto Id() const noexcept -> ActionIdentifier const& { return id_; } [[nodiscard]] auto Command() && noexcept -> std::vector { return std::move(command_); } [[nodiscard]] auto Command() const& noexcept -> std::vector const& { return command_; } [[nodiscard]] auto Cwd() const noexcept -> std::string const& { return cwd_; } [[nodiscard]] auto Env() const& noexcept -> std::map const& { return env_; } [[nodiscard]] auto Env() && noexcept -> std::map { return std::move(env_); } [[nodiscard]] auto IsTreeAction() const noexcept -> bool { return is_tree_; } [[nodiscard]] auto IsTreeOverlayAction() const noexcept -> bool { return is_tree_overlay_; } [[nodiscard]] auto IsOverlayDisjoint() const noexcept -> bool { return overlay_disjoint_; } [[nodiscard]] auto MayFail() const noexcept -> std::optional const& { return may_fail_; } [[nodiscard]] auto NoCache() const noexcept -> bool { return no_cache_; } [[nodiscard]] auto TimeoutScale() const noexcept -> double { return timeout_scale_; } [[nodiscard]] auto ExecutionProperties() const& noexcept -> std::map const& { return execution_properties_; } [[nodiscard]] auto ExecutionProperties() && noexcept -> std::map { return std::move(execution_properties_); } [[nodiscard]] static auto CreateTreeAction( ActionIdentifier const& id) noexcept -> Action { return Action{ id, /*is_tree_overlay=*/false, /*overlay_disjoint=*/false}; } [[nodiscard]] static auto CreateTreeOverlayAction( ActionIdentifier const& id, bool disjoint) noexcept -> Action { return Action{id, /*is_tree_overlay=*/true, disjoint}; } private: ActionIdentifier id_; std::vector command_; std::string cwd_; std::map env_; bool is_tree_{}; bool is_tree_overlay_{}; bool overlay_disjoint_{}; std::optional may_fail_; bool no_cache_{}; double timeout_scale_{}; std::map execution_properties_; explicit Action(ActionIdentifier id, bool is_tree_overlay, bool overlay_disjoint) noexcept : id_{std::move(id)}, is_tree_{not is_tree_overlay}, is_tree_overlay_{is_tree_overlay}, overlay_disjoint_{overlay_disjoint} {} }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/action_description.hpp000066400000000000000000000233011516554100600302630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP #include #include #include #include #include #include #include #include // std::move #include #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/json.hpp" class ActionDescription { public: using outputs_t = std::vector; using inputs_t = std::unordered_map; using Ptr = std::shared_ptr; ActionDescription(outputs_t output_files, outputs_t output_dirs, Action action, inputs_t inputs) : output_files_{std::move(output_files)}, output_dirs_{std::move(output_dirs)}, action_{std::move(action)}, inputs_{std::move(inputs)} {} [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, std::string const& id, nlohmann::json const& desc) noexcept -> std::optional { try { auto outputs = ExtractValueAs>(desc, "output"); auto output_dirs = ExtractValueAs>(desc, "output_dirs"); auto command = ExtractValueAs>(desc, "command"); if ((not outputs.has_value() or outputs->empty()) and (not output_dirs.has_value() or output_dirs->empty())) { Logger::Log( LogLevel::Error, "Action description for action \"{}\" incomplete: values " "for either \"output\" or \"output_dir\" must be non-empty " "array.", id); return std::nullopt; } if (not command.has_value() or command->empty()) { Logger::Log( LogLevel::Error, "Action description for action \"{}\" incomplete: values " "for \"command\" must be non-empty array.", id); return std::nullopt; } if (not outputs) { outputs = std::vector{}; } if (not output_dirs) { output_dirs = std::vector{}; } std::string cwd{}; auto cwd_it = desc.find("cwd"); if (cwd_it != desc.end()) { if (cwd_it->is_string()) { cwd = *cwd_it; } else { Logger::Log(LogLevel::Error, "cwd, if given, has to be a string"); return std::nullopt; } } auto optional_key_value_reader = [](nlohmann::json const& action_desc, std::string const& key) -> nlohmann::json { auto it = action_desc.find(key); if (it == action_desc.end()) { return nlohmann::json::object(); } return *it; }; auto const input = optional_key_value_reader(desc, "input"); auto const env = optional_key_value_reader(desc, "env"); if (not(input.is_object() and env.is_object())) { Logger::Log( LogLevel::Error, "Action description for action \"{}\" type error: values " "for \"input\" and \"env\" must be objects.", id); return std::nullopt; } inputs_t inputs{}; for (auto const& [path, input_desc] : input.items()) { auto artifact = ArtifactDescription::FromJson(hash_type, input_desc); if (not artifact) { return std::nullopt; } inputs.emplace(path, std::move(*artifact)); } std::optional may_fail{}; bool no_cache{}; auto may_fail_it = desc.find("may_fail"); if (may_fail_it != desc.end()) { if (may_fail_it->is_null()) { may_fail = std::nullopt; } else if (may_fail_it->is_string()) { may_fail = *may_fail_it; } else { Logger::Log(LogLevel::Error, "may_fail has to be a null or a string"); return std::nullopt; } } auto no_cache_it = desc.find("no_cache"); if (no_cache_it != desc.end()) { if (not no_cache_it->is_boolean()) { Logger::Log(LogLevel::Error, "no_cache has to be a boolean"); return std::nullopt; } no_cache = *no_cache_it; } double timeout_scale{1.0}; auto timeout_scale_it = desc.find("timeout scaling"); if (timeout_scale_it != desc.end()) { if (not timeout_scale_it->is_number()) { Logger::Log(LogLevel::Error, "timeout scaling has to be a number"); } timeout_scale = *timeout_scale_it; } auto const execution_properties = optional_key_value_reader(desc, "execution properties"); if (not execution_properties.is_object()) { Logger::Log(LogLevel::Error, "Execution properties have to a map"); return std::nullopt; } return std::make_shared( std::move(*outputs), std::move(*output_dirs), Action{id, std::move(*command), cwd, env, may_fail, no_cache, timeout_scale, execution_properties}, inputs); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Failed to parse action description from JSON with error:\n{}", ex.what()); } return std::nullopt; } [[nodiscard]] auto Id() const noexcept -> ActionIdentifier { return action_.Id(); } [[nodiscard]] auto ToJson() const -> nlohmann::json { auto json = nlohmann::json{{"command", action_.Command()}}; if (not output_files_.empty()) { json["output"] = output_files_; } if (not output_dirs_.empty()) { json["output_dirs"] = output_dirs_; } if (not inputs_.empty()) { auto inputs = nlohmann::json::object(); for (auto const& [path, artifact] : inputs_) { inputs[path] = artifact.ToJson(); } json["input"] = inputs; } if (not action_.Env().empty()) { json["env"] = action_.Env(); } if (action_.MayFail()) { json["may_fail"] = *action_.MayFail(); } if (action_.NoCache()) { json["no_cache"] = true; } if (action_.TimeoutScale() != 1.0) { json["timeout scaling"] = action_.TimeoutScale(); } if (not action_.Cwd().empty()) { json["cwd"] = action_.Cwd(); } if (not action_.ExecutionProperties().empty()) { json["execution properties"] = action_.ExecutionProperties(); } return json; } [[nodiscard]] auto OutputFiles() const& noexcept -> outputs_t const& { return output_files_; } [[nodiscard]] auto OutputFiles() && noexcept -> outputs_t { return std::move(output_files_); } [[nodiscard]] auto OutputDirs() const& noexcept -> outputs_t const& { return output_dirs_; } [[nodiscard]] auto OutputDirs() && noexcept -> outputs_t { return std::move(output_dirs_); } [[nodiscard]] auto GraphAction() const& noexcept -> Action const& { return action_; } [[nodiscard]] auto GraphAction() && noexcept -> Action { return std::move(action_); } [[nodiscard]] auto Inputs() const& noexcept -> inputs_t const& { return inputs_; } [[nodiscard]] auto Inputs() && noexcept -> inputs_t { return std::move(inputs_); } private: outputs_t output_files_; outputs_t output_dirs_; Action action_; inputs_t inputs_; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact.hpp000066400000000000000000000176171516554100600262150ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hash_combine.hpp" // Artifacts (source files, libraries, executables...) need to store their // identifier class Artifact { public: struct ObjectInfo { ArtifactDigest digest; ObjectType type{}; bool failed{}; [[nodiscard]] auto operator==(ObjectInfo const& other) const { return (digest == other.digest and type == other.type and failed == other.failed); } [[nodiscard]] auto operator!=(ObjectInfo const& other) const { return not(*this == other); } // Create string of the form '[hash:size:type]' [[nodiscard]] auto ToString(bool size_unknown = false) const noexcept -> std::string { auto size_str = size_unknown ? std::string{} : std::to_string(digest.size()); return fmt::format("[{}:{}:{}]{}", digest.hash(), size_str, ToChar(type), failed ? " FAILED" : ""); } // Create JSON of the form '{"id": "hash", "size": x, "file_type": "f"}' // As the failed property is only internal to a run, discard it. [[nodiscard]] auto ToJson() const -> nlohmann::json { return {{"id", digest.hash()}, {"size", digest.size()}, {"file_type", std::string{ToChar(type)}}}; } [[nodiscard]] static auto FromString(HashFunction::Type hash_type, std::string const& s) noexcept -> std::optional { std::istringstream iss(s); std::string id{}; std::string size_str{}; std::string type{}; if (not(iss.get() == '[') or not std::getline(iss, id, ':') or not std::getline(iss, size_str, ':') or not std::getline(iss, type, ']')) { Logger::Log(LogLevel::Debug, "failed parsing object info from string."); return std::nullopt; } std::size_t size = 0; try { size = std::stoul(size_str); } catch (std::out_of_range const& e) { Logger::Log(LogLevel::Debug, "size raised out_of_range exception."); return std::nullopt; } catch (std::invalid_argument const& e) { Logger::Log(LogLevel::Debug, "size raised invalid_argument exception."); return std::nullopt; } auto const object_type = FromChar(*type.c_str()); auto digest = ArtifactDigestFactory::Create( hash_type, id, size, IsTreeObject(object_type)); if (not digest) { Logger::Log(LogLevel::Debug, "{}", std::move(digest).error()); return std::nullopt; } return ObjectInfo{.digest = *std::move(digest), .type = object_type}; } }; explicit Artifact(ArtifactIdentifier id) noexcept : id_{std::move(id)} {} [[nodiscard]] auto Id() const& noexcept -> ArtifactIdentifier const& { return id_; } [[nodiscard]] auto Id() && noexcept -> ArtifactIdentifier { return std::move(id_); } [[nodiscard]] auto FilePath() const noexcept -> std::optional { return file_path_; } [[nodiscard]] auto Repository() const noexcept -> std::string { return repo_; } [[nodiscard]] auto Digest() const noexcept -> std::optional { return object_info_ ? std::optional{object_info_->digest} : std::nullopt; } [[nodiscard]] auto Type() const noexcept -> std::optional { return object_info_ ? std::optional{object_info_->type} : std::nullopt; } [[nodiscard]] auto Info() const& noexcept -> std::optional const& { return object_info_; } [[nodiscard]] auto Info() && noexcept -> std::optional { return std::move(object_info_); } void SetObjectInfo(ObjectInfo const& object_info, bool fail_info) const noexcept { if (fail_info) { object_info_ = ObjectInfo{.digest = object_info.digest, .type = object_info.type, .failed = true}; } else { object_info_ = object_info; } } void SetObjectInfo(ArtifactDigest const& digest, ObjectType type, bool failed) const noexcept { object_info_ = ObjectInfo{.digest = digest, .type = type, .failed = failed}; } [[nodiscard]] static auto CreateLocalArtifact( std::string const& id, std::filesystem::path const& file_path, std::string const& repo) noexcept -> Artifact { return Artifact{id, file_path, repo}; } [[nodiscard]] static auto CreateKnownArtifact( std::string const& id, ArtifactDigest const& digest, ObjectType type, std::optional const& repo) noexcept -> Artifact { return Artifact{id, digest, type, false, repo}; } [[nodiscard]] static auto CreateActionArtifact( std::string const& id) noexcept -> Artifact { return Artifact{id}; } private: ArtifactIdentifier id_; std::optional file_path_; std::string repo_; mutable std::optional object_info_; Artifact(ArtifactIdentifier id, std::filesystem::path const& file_path, std::string repo) noexcept : id_{std::move(id)}, file_path_{file_path}, repo_{std::move(repo)} {} Artifact(ArtifactIdentifier id, ArtifactDigest const& digest, ObjectType type, bool failed, std::optional repo) noexcept : id_{std::move(id)} { SetObjectInfo(digest, type, failed); if (repo) { repo_ = std::move(*repo); } } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()( Artifact::ObjectInfo const& info) const noexcept -> std::size_t { std::size_t seed{}; hash_combine(&seed, info.digest); hash_combine(&seed, info.type); hash_combine(&seed, info.failed); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_blob.cpp000066400000000000000000000212401516554100600271710ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/artifact_blob.hpp" #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/in_place_visitor.hpp" namespace { [[nodiscard]] auto ReadFromFile(std::filesystem::path const& file) noexcept -> std::shared_ptr { auto content = FileSystemManager::ReadFile(file); if (not content.has_value()) { return nullptr; } return std::make_shared(*std::move(content)); } } // namespace auto ArtifactBlob::FromMemory(HashFunction hash_function, ObjectType type, std::string content) noexcept -> expected { try { auto digest = IsTreeObject(type) ? ArtifactDigestFactory::HashDataAs( hash_function, content) : ArtifactDigestFactory::HashDataAs( hash_function, content); return ArtifactBlob{ std::move(digest), std::make_shared(std::move(content)), IsExecutableObject(type)}; } catch (const std::exception& e) { return unexpected{fmt::format( "ArtifactBlob::FromMemory: Got an exception:\n{}", e.what())}; } catch (...) { return unexpected{ "ArtifactBlob::FromMemory: Got an unknown exception"}; } } auto ArtifactBlob::FromFile(HashFunction hash_function, ObjectType type, std::filesystem::path file) noexcept -> expected { try { if (not FileSystemManager::IsFile(file)) { return unexpected{ fmt::format("ArtifactBlob::FromFile: Not a regular file:\n{}", file.string())}; } auto digest = IsTreeObject(type) ? ArtifactDigestFactory::HashFileAs( hash_function, file) : ArtifactDigestFactory::HashFileAs( hash_function, file); if (not digest.has_value()) { return unexpected{fmt::format( "ArtifactBlob::FromFile: Failed to hash {}", file.string())}; } return ArtifactBlob{ *std::move(digest), std::move(file), IsExecutableObject(type)}; } catch (const std::exception& e) { return unexpected{fmt::format( "ArtifactBlob::FromFile: Got an exception while processing {}:\n{}", file.string(), e.what())}; } catch (...) { return unexpected{ fmt::format("ArtifactBlob::FromFile: Got an unknown exception " "while processing {}", file.string())}; } } auto ArtifactBlob::FromTempFile(HashFunction hash_function, ObjectType type, TmpFile::Ptr file) noexcept -> expected { try { if (file == nullptr) { return unexpected{ "ArtifactBlob::CreateFromTempFile: missing temp file."}; } auto digest = IsTreeObject(type) ? ArtifactDigestFactory::HashFileAs( hash_function, file->GetPath()) : ArtifactDigestFactory::HashFileAs( hash_function, file->GetPath()); if (not digest.has_value()) { return unexpected{fmt::format( "ArtifactBlob::CreateFromTempFile: Failed to hash {}", file->GetPath().string())}; } return ArtifactBlob{ *std::move(digest), std::move(file), IsExecutableObject(type)}; } catch (const std::exception& e) { return unexpected{fmt::format( "ArtifactBlob::FromTempFile: Got an exception:\n{}", e.what())}; } catch (...) { return unexpected{ "ArtifactBlob::FromTempFile: Got an unknown exception."}; } } auto ArtifactBlob::FromTempFile(HashFunction hash_type, ObjectType type, TmpDir::Ptr const& temp_space, std::string const& content) noexcept -> expected { try { if (temp_space == nullptr) { return unexpected{ "ArtifactBlob::FromTempFile: missing temp space."}; } auto file = TmpDir::CreateFile(temp_space); if (file == nullptr) { return unexpected{ "ArtifactBlob::FromTempFile: failed to create a new temp " "file."}; } if (not FileSystemManager::WriteFile(content, file->GetPath())) { return unexpected{ "ArtifactBlob::FromTempFile: failed to write content to a " "temp file."}; } return FromTempFile(hash_type, type, std::move(file)); } catch (const std::exception& e) { return unexpected{fmt::format( "ArtifactBlob::FromTempFile: Got an exception:\n{}", e.what())}; } catch (...) { return unexpected{ "ArtifactBlob::FromTempFile: Got an unknown exception."}; } } auto ArtifactBlob::ReadContent() const noexcept -> std::shared_ptr { using Result = std::shared_ptr; static constexpr InPlaceVisitor kVisitor{ [](InMemory const& value) -> Result { return value; }, [](InFile const& value) -> Result { return ::ReadFromFile(value); }, [](InTempFile const& value) -> Result { return ::ReadFromFile(value->GetPath()); }, }; try { return std::visit(kVisitor, content_); } catch (...) { return nullptr; } } auto ArtifactBlob::ReadIncrementally(std::size_t chunk_size) const& noexcept -> expected { using Result = expected; const InPlaceVisitor visitor{ [chunk_size](InMemory const& value) -> Result { if (value == nullptr) { return unexpected{ "ArtifactBlob::ReadIncrementally: missing memory source"}; } return IncrementalReader::FromMemory(chunk_size, value.get()); }, [chunk_size](InFile const& value) -> Result { return IncrementalReader::FromFile(chunk_size, value); }, [chunk_size](InTempFile const& value) -> Result { return IncrementalReader::FromFile(chunk_size, value->GetPath()); }, }; try { return std::visit(visitor, content_); } catch (std::exception const& e) { return unexpected{fmt::format( "ArtifactBlob::ReadIncrementally: Got an exception:\n{}", e.what())}; } catch (...) { return unexpected{ "ArtifactBlob::ReadIncrementally: Got an unknown exception"}; } } auto ArtifactBlob::GetFilePath() const& noexcept -> std::optional { using Result = std::optional; static constexpr InPlaceVisitor kVisitor{ [](InMemory const&) -> Result { return std::nullopt; }, [](InFile const& value) -> Result { return value; }, [](InTempFile const& value) -> Result { return value->GetPath(); }, }; try { return std::visit(kVisitor, content_); } catch (...) { return std::nullopt; } } namespace std { auto hash::operator()(ArtifactBlob const& blob) const noexcept -> std::size_t { std::size_t seed{}; hash_combine(&seed, blob.GetDigest()); hash_combine(&seed, blob.IsExecutable()); return seed; } } // namespace std just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_blob.hpp000066400000000000000000000150241516554100600272010ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_BLOB_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_BLOB_HPP #include #include #include #include #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/incremental_reader.hpp" #include "src/utils/cpp/tmp_dir.hpp" class ArtifactBlob final { public: /// \brief Create ArtifactBlob and keep the given content in memory. The /// content is hashed based on the given hash function and ObjectType. /// \param hash_function Hash function that must be used for hashing. /// \param type Type of the content. /// \param content String to be stored /// \return Valid ArtifactBlob on success or an error message on failure. [[nodiscard]] static auto FromMemory(HashFunction hash_function, ObjectType type, std::string content) noexcept -> expected; /// \brief Create ArtifactBlob based on the existing file. The content is /// hashed based on the given hash function and ObjectType. /// \param hash_function Hash function that must be used for hashing. /// \param type Type of the content. /// \param file Existing file to be used as the source of /// content. /// \return Valid ArtifactBlob on success or an error message on failure. [[nodiscard]] static auto FromFile(HashFunction hash_function, ObjectType type, std::filesystem::path file) noexcept -> expected; /// \brief Create ArtifactBlob based on the existing temporary file. The /// content is hashed based on the given hash function and ObjectType. /// \param hash_function HashFunction that must be used for hashing. /// \param type Type of the content. /// \param file Temporary file to be used as the source of /// content. /// \return Valid ArtifactBlob on success or an error message on failure. [[nodiscard]] static auto FromTempFile(HashFunction hash_function, ObjectType type, TmpFile::Ptr file) noexcept -> expected; /// \brief Create ArtifactBlob and write the given content to the temporary /// space. The content is hashed based on the given hash function and /// ObjectType. /// \param hash_function HashFunction that must be used for hashing. /// \param type Type of the content. /// \param temp_space Temporary space where a new temporary file may /// be created. /// \param content Content to be stored in the temporary file. /// \return Valid ArtifactBlob on success or an error message on failure. [[nodiscard]] static auto FromTempFile(HashFunction hash_type, ObjectType type, TmpDir::Ptr const& temp_space, std::string const& content) noexcept -> expected; [[nodiscard]] auto operator==(ArtifactBlob const& other) const noexcept -> bool { return digest_ == other.digest_ and is_executable_ == other.is_executable_; } /// \brief Obtain the digest of the content. [[nodiscard]] auto GetDigest() const noexcept -> ArtifactDigest const& { return digest_; } /// \brief Obtain the size of the content. [[nodiscard]] auto GetContentSize() const noexcept -> std::size_t { return digest_.size(); } /// \brief Read the content from source. This operation may result in the /// entire file being read into memory. [[nodiscard]] auto ReadContent() const noexcept -> std::shared_ptr; /// \brief Create an IncrementalReader that uses this ArtifactBlob's content /// source. /// \param chunk_size Size of chunk, must be greater than 0. /// \return Valid IncrementalReader on success or an error message on /// failure. [[nodiscard]] auto ReadIncrementally(std::size_t chunk_size) const& noexcept -> expected; /// \brief Obtain the path to the file that is used as the content source. /// If ArtifactBlob doesn't use a filesystem source or an internal error /// occurs, std::nullopt is returned. [[nodiscard]] auto GetFilePath() const& noexcept -> std::optional; /// \brief Set executable permission. void SetExecutable(bool is_executable) noexcept { is_executable_ = is_executable; } /// \brief Obtain executable permission. [[nodiscard]] auto IsExecutable() const noexcept -> bool { return is_executable_; } private: using InMemory = std::shared_ptr; using InFile = std::filesystem::path; using InTempFile = TmpFile::Ptr; using ContentSource = std::variant; ArtifactDigest digest_; ContentSource content_; bool is_executable_; explicit ArtifactBlob(ArtifactDigest digest, ContentSource content, bool is_executable) noexcept : digest_{std::move(digest)}, content_{std::move(content)}, is_executable_{is_executable} {} }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(ArtifactBlob const& blob) const noexcept -> std::size_t; }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_BLOB_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_description.cpp000066400000000000000000000342271516554100600306070ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/artifact_description.hpp" #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/json.hpp" namespace { [[nodiscard]] auto DescribeLocalArtifact(std::filesystem::path const& src_path, std::string const& repository) -> nlohmann::json; [[nodiscard]] auto DescribeKnownArtifact(std::string const& blob_id, std::size_t size, ObjectType type = ObjectType::File) -> nlohmann::json; [[nodiscard]] auto DescribeActionArtifact(std::string const& action_id, std::string const& out_path) -> nlohmann::json; [[nodiscard]] auto DescribeTreeArtifact(std::string const& tree_id) -> nlohmann::json; [[nodiscard]] auto DescribeTreeOverlayArtifact( std::string const& tree_overlay_id) -> nlohmann::json; [[nodiscard]] auto CreateLocalArtifactDescription(nlohmann::json const& data) -> std::optional; [[nodiscard]] auto CreateKnownArtifactDescription(HashFunction::Type hash_type, nlohmann::json const& data) -> std::optional; [[nodiscard]] auto CreateActionArtifactDescription(nlohmann::json const& data) -> std::optional; [[nodiscard]] auto CreateTreeArtifactDescription(nlohmann::json const& data) -> std::optional; [[nodiscard]] auto CreateTreeOverlayArtifactDescription( nlohmann::json const& data) -> std::optional; } // namespace auto ArtifactDescription::CreateLocal(std::filesystem::path path, std::string repository) noexcept -> ArtifactDescription { Local data{std::move(path), std::move(repository)}; return ArtifactDescription{std::move(data)}; } auto ArtifactDescription::CreateAction(std::string action_id, std::filesystem::path path) noexcept -> ArtifactDescription { Action data{std::move(action_id), std::move(path)}; return ArtifactDescription{std::move(data)}; } auto ArtifactDescription::CreateKnown(ArtifactDigest digest, ObjectType file_type, std::optional repo) noexcept -> ArtifactDescription { Known data{std::move(digest), file_type, std::move(repo)}; return ArtifactDescription{std::move(data)}; } auto ArtifactDescription::CreateTree(std::string tree_id) noexcept -> ArtifactDescription { return ArtifactDescription{Tree{std::move(tree_id)}}; } auto ArtifactDescription::CreateTreeOverlay( std::string tree_overlay_id) noexcept -> ArtifactDescription { return ArtifactDescription{TreeOverlay{std::move(tree_overlay_id)}}; } auto ArtifactDescription::FromJson(HashFunction::Type hash_type, nlohmann::json const& json) noexcept -> std::optional { try { auto const type = ExtractValueAs( json, "type", [](std::string const& error) { Logger::Log( LogLevel::Error, "{}\ncan not retrieve value for \"type\" from artifact " "description.", error); }); auto const data = ExtractValueAs( json, "data", [](std::string const& error) { Logger::Log( LogLevel::Error, "{}\ncan not retrieve value for \"data\" from artifact " "description.", error); }); if (not(type and data)) { return std::nullopt; } if (*type == "LOCAL") { return CreateLocalArtifactDescription(*data); } if (*type == "KNOWN") { return CreateKnownArtifactDescription(hash_type, *data); } if (*type == "ACTION") { return CreateActionArtifactDescription(*data); } if (*type == "TREE") { return CreateTreeArtifactDescription(*data); } if (*type == "TREE_OVERLAY") { return CreateTreeOverlayArtifactDescription(*data); } Logger::Log(LogLevel::Error, R"(artifact type must be one of "LOCAL", "KNOWN", "ACTION", "TREE", or "TREE_OVERLAY")"); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Failed to parse artifact description from JSON with " "error:\n{}", ex.what()); } return std::nullopt; } auto ArtifactDescription::ToJson() const -> nlohmann::json { if (std::holds_alternative(data_)) { auto const& [path, repo] = std::get(data_); return DescribeLocalArtifact(path.string(), repo); } if (std::holds_alternative(data_)) { auto const& [digest, file_type, _] = std::get(data_); return DescribeKnownArtifact(digest.hash(), digest.size(), file_type); } if (std::holds_alternative(data_)) { auto const& [action_id, path] = std::get(data_); return DescribeActionArtifact(action_id, path); } if (std::holds_alternative(data_)) { return DescribeTreeArtifact(std::get(data_).tree); } if (std::holds_alternative(data_)) { return DescribeTreeOverlayArtifact( std::get(data_).tree_overlay); } Logger::Log(LogLevel::Error, "Internal error, unknown artifact type"); Ensures(false); // unreachable return {}; } auto ArtifactDescription::ToArtifact() const noexcept -> Artifact { try { if (std::holds_alternative(data_)) { auto const& [path, repo] = std::get(data_); return Artifact::CreateLocalArtifact(id_, path.string(), repo); } if (std::holds_alternative(data_)) { auto const& [digest, file_type, repo] = std::get(data_); return Artifact::CreateKnownArtifact(id_, digest, file_type, repo); } if (std::holds_alternative(data_) or std::holds_alternative(data_) or std::holds_alternative(data_)) { return Artifact::CreateActionArtifact(id_); } Logger::Log(LogLevel::Error, "Internal error, unknown artifact type"); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Creating artifact failed with error:\n{}", ex.what()); } Ensures(false); // unreachable return Artifact{{}}; } auto ArtifactDescription::ToString(int indent) const noexcept -> std::string { try { return ToJson().dump(indent); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Serializing artifact failed with error:\n{}", ex.what()); } return {}; } auto ArtifactDescription::ComputeId(nlohmann::json const& desc) noexcept -> ArtifactIdentifier { try { // The type of HashFunction is irrelevant here. It is used for // identification and quick comparison of descriptions. SHA256 is used. return HashFunction{HashFunction::Type::PlainSHA256} .PlainHashData(desc.dump()) .Bytes(); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Computing artifact id failed with error:\n{}", ex.what()); } return {}; } namespace { auto DescribeLocalArtifact(std::filesystem::path const& src_path, std::string const& repository) -> nlohmann::json { return { {"type", "LOCAL"}, {"data", {{"path", src_path.string()}, {"repository", repository}}}}; } auto DescribeKnownArtifact(std::string const& blob_id, std::size_t size, ObjectType type) -> nlohmann::json { std::string const typestr{ToChar(type)}; return { {"type", "KNOWN"}, {"data", {{"id", blob_id}, {"size", size}, {"file_type", typestr}}}}; } auto DescribeActionArtifact(std::string const& action_id, std::string const& out_path) -> nlohmann::json { return {{"type", "ACTION"}, {"data", {{"id", action_id}, {"path", out_path}}}}; } auto DescribeTreeArtifact(std::string const& tree_id) -> nlohmann::json { return {{"type", "TREE"}, {"data", {{"id", tree_id}}}}; } auto DescribeTreeOverlayArtifact(std::string const& tree_overlay_id) -> nlohmann::json { return {{"type", "TREE_OVERLAY"}, {"data", {{"id", tree_overlay_id}}}}; } auto CreateLocalArtifactDescription(nlohmann::json const& data) -> std::optional { auto path = ExtractValueAs(data, "path", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"path\" from " "LOCAL artifact's data.", error); }); auto repository = ExtractValueAs( data, "repository", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"path\" from " "LOCAL artifact's data.", error); }); if (path.has_value() and repository.has_value()) { return ArtifactDescription::CreateLocal(std::move(*path), std::move(*repository)); } return std::nullopt; } auto CreateKnownArtifactDescription(HashFunction::Type hash_type, nlohmann::json const& data) -> std::optional { auto const blob_id = ExtractValueAs(data, "id", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"id\" from " "KNOWN artifact's data.", error); }); auto const size = ExtractValueAs(data, "size", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"size\" from " "KNOWN artifact's data.", error); }); auto const file_type = ExtractValueAs( data, "file_type", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"file_type\" from " "KNOWN artifact's data.", error); }); if (blob_id.has_value() and size.has_value() and file_type.has_value() and file_type->size() == 1) { auto const object_type = FromChar((*file_type)[0]); auto digest = ArtifactDigestFactory::Create( hash_type, *blob_id, *size, IsTreeObject(object_type)); if (not digest) { return std::nullopt; } return ArtifactDescription::CreateKnown(*std::move(digest), object_type); } return std::nullopt; } auto CreateActionArtifactDescription(nlohmann::json const& data) -> std::optional { auto action_id = ExtractValueAs(data, "id", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"id\" from " "ACTION artifact's data.", error); }); auto path = ExtractValueAs(data, "path", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"path\" from " "ACTION artifact's data.", error); }); if (action_id.has_value() and path.has_value()) { return ArtifactDescription::CreateAction(std::move(*action_id), std::move(*path)); } return std::nullopt; } auto CreateTreeArtifactDescription(nlohmann::json const& data) -> std::optional { auto tree_id = ExtractValueAs(data, "id", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"id\" from " "TREE artifact's data.", error); }); if (tree_id.has_value()) { return ArtifactDescription::CreateTree(std::move(*tree_id)); } return std::nullopt; } auto CreateTreeOverlayArtifactDescription(nlohmann::json const& data) -> std::optional { auto tree_overlay_id = ExtractValueAs(data, "id", [](std::string const& error) { Logger::Log(LogLevel::Error, "{}\ncan not retrieve value for \"id\" from " "TREE_OVERLAY artifact's data.", error); }); if (tree_overlay_id.has_value()) { return ArtifactDescription::CreateTreeOverlay( std::move(*tree_overlay_id)); } return std::nullopt; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_description.hpp000066400000000000000000000102121516554100600306000ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP #include #include #include #include #include #include // std::move #include #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" class ArtifactDescription final { using Local = std::pair; using Known = std::tuple>; using Action = std::pair; public: struct Tree { std::string tree; }; struct TreeOverlay { std::string tree_overlay; }; [[nodiscard]] static auto CreateLocal(std::filesystem::path path, std::string repository) noexcept -> ArtifactDescription; [[nodiscard]] static auto CreateAction(std::string action_id, std::filesystem::path path) noexcept -> ArtifactDescription; [[nodiscard]] static auto CreateKnown( ArtifactDigest digest, ObjectType file_type, std::optional repo = std::nullopt) noexcept -> ArtifactDescription; [[nodiscard]] static auto CreateTree(std::string tree_id) noexcept -> ArtifactDescription; [[nodiscard]] static auto CreateTreeOverlay( std::string tree_overlay_id) noexcept -> ArtifactDescription; [[nodiscard]] auto Id() const& noexcept -> ArtifactIdentifier const& { return id_; } [[nodiscard]] auto Id() && noexcept -> ArtifactIdentifier { return std::move(id_); } [[nodiscard]] auto IsKnown() const noexcept -> bool { return std::holds_alternative(data_); } [[nodiscard]] auto IsTree() const noexcept -> bool { return std::holds_alternative(data_); } [[nodiscard]] auto IsTreeOverlay() const noexcept -> bool { return std::holds_alternative(data_); } [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, nlohmann::json const& json) noexcept -> std::optional; [[nodiscard]] auto ToJson() const -> nlohmann::json; [[nodiscard]] auto ToArtifact() const noexcept -> Artifact; [[nodiscard]] auto ToString(int indent = 0) const noexcept -> std::string; [[nodiscard]] auto operator==( ArtifactDescription const& other) const noexcept -> bool { return id_ == other.id_; } [[nodiscard]] auto operator!=( ArtifactDescription const& other) const noexcept -> bool { return not(*this == other); } private: std::variant data_; ArtifactIdentifier id_; template explicit ArtifactDescription(T data) noexcept : data_{std::move(data)}, id_{ComputeId(ToJson())} {} [[nodiscard]] static auto ComputeId(nlohmann::json const& desc) noexcept -> ArtifactIdentifier; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(ArtifactDescription const& a) const { return std::hash{}(a.Id()); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_digest.hpp000066400000000000000000000045641516554100600275510ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP #define INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP #include #include #include #include // std::move #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/utils/cpp/hash_combine.hpp" // Provides getter for size with convenient non-protobuf type. Contains an // unprefixed hex string as hash. class ArtifactDigest final { friend class ArtifactDigestFactory; public: ArtifactDigest() noexcept = default; explicit ArtifactDigest(HashInfo hash_info, std::size_t size) noexcept : hash_info_{std::move(hash_info)}, size_{size} {} [[nodiscard]] auto hash() const& noexcept -> std::string const& { return hash_info_.Hash(); } [[nodiscard]] auto hash() && noexcept -> std::string { return std::move(hash_info_).Hash(); } [[nodiscard]] auto size() const noexcept -> std::size_t { return size_; } [[nodiscard]] auto IsTree() const noexcept -> bool { return hash_info_.IsTree(); } [[nodiscard]] auto operator==(ArtifactDigest const& other) const -> bool { return hash_info_ == other.hash_info_; } [[nodiscard]] auto GetHashType() const& noexcept -> HashFunction::Type { return hash_info_.HashType(); } private: HashInfo hash_info_; std::size_t size_ = 0; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(ArtifactDigest const& digest) const noexcept -> std::size_t { std::size_t seed{}; hash_combine(&seed, digest.hash()); hash_combine(&seed, digest.IsTree()); return seed; } }; } // namespace std #endif // INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_digest_factory.cpp000066400000000000000000000042201516554100600312600ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/artifact_digest_factory.hpp" #include #include "gsl/gsl" #include "src/buildtool/common/bazel_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" auto ArtifactDigestFactory::Create(HashFunction::Type hash_type, std::string hash, std::size_t size, bool is_tree) noexcept -> expected { auto hash_info = HashInfo::Create(hash_type, std::move(hash), ProtocolTraits::IsTreeAllowed(hash_type) and is_tree); if (not hash_info) { return unexpected{std::move(hash_info).error()}; } return ArtifactDigest{*std::move(hash_info), size}; } auto ArtifactDigestFactory::FromBazel(HashFunction::Type hash_type, bazel_re::Digest const& digest) noexcept -> expected { auto hash_info = BazelDigestFactory::ToHashInfo(hash_type, digest); if (not hash_info) { return unexpected{std::move(hash_info).error()}; } return ArtifactDigest{*std::move(hash_info), static_cast(digest.size_bytes())}; } auto ArtifactDigestFactory::ToBazel(ArtifactDigest const& digest) -> bazel_re::Digest { return BazelDigestFactory::Create(digest.hash_info_, gsl::narrow(digest.size_)); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/artifact_digest_factory.hpp000066400000000000000000000102721516554100600312710ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DIGEST_FACTORY_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DIGEST_FACTORY_HPP #include #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" namespace build::bazel::remote::execution::v2 { class Digest; } namespace bazel_re = build::bazel::remote::execution::v2; class ArtifactDigestFactory final { public: /// \brief Create ArtifactDigest from plain hash. /// \param hash_type Type of the hash function that was used for creation /// of the hash /// \param hash Hexadecimal plain hash /// \param size Size of the content /// \return A valid ArtifactDigest on success or an error message if /// validation fails. [[nodiscard]] static auto Create(HashFunction::Type hash_type, std::string hash, std::size_t size, bool is_tree) noexcept -> expected; /// \brief Create ArtifactDigest from bazel_re::Digest /// \param hash_type Type of the hash function that was used for creation of /// the hash /// \param digest Digest to be converted /// \return A valid ArtifactDigest on success or an error message if /// validation fails. [[nodiscard]] static auto FromBazel(HashFunction::Type hash_type, bazel_re::Digest const& digest) noexcept -> expected; /// \brief Convert ArtifactDigest to bazel_re::Digest. Throws an exception /// on a narrow conversion error. /// \param digest Digest to be converted. /// \return A valid bazel_re::Digest [[nodiscard]] static auto ToBazel(ArtifactDigest const& digest) -> bazel_re::Digest; /// \brief Hash content using hash function and return a valid /// ArtifactDigest /// \tparam kType Type of the hashing algorithm to be used /// \param hash_function Hash function to be used for hashing /// \param content Content to be hashed /// \return The digest of the content template [[nodiscard]] static auto HashDataAs(HashFunction hash_function, std::string const& content) noexcept -> ArtifactDigest { auto hash_info = HashInfo::HashData(hash_function, content, IsTreeObject(kType)); return ArtifactDigest{std::move(hash_info), content.size()}; } /// \brief Hash file using hash function and return a valid ArtifactDigest /// \tparam kType Type of the hashing algorithm to be used /// \param hash_function Hash function to be used for hashing /// \param content Content to be hashed /// \return The digest of the file template [[nodiscard]] static auto HashFileAs( HashFunction hash_function, std::filesystem::path const& path) noexcept -> std::optional { auto hash_info = HashInfo::HashFile(hash_function, path, IsTreeObject(kType)); if (not hash_info) { return std::nullopt; } return ArtifactDigest{std::move(hash_info->first), static_cast(hash_info->second)}; } }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DIGEST_FACTORY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/bazel_digest_factory.cpp000066400000000000000000000037201516554100600305640ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/bazel_digest_factory.hpp" #include #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hasher.hpp" auto BazelDigestFactory::Create(HashInfo const& hash_info, std::int64_t size) noexcept -> bazel_re::Digest { auto hash = ProtocolTraits::IsNative(hash_info.HashType()) ? Prefix(hash_info.Hash(), hash_info.IsTree()) : hash_info.Hash(); bazel_re::Digest digest{}; digest.set_hash(std::move(hash)); digest.set_size_bytes(size); return digest; } auto BazelDigestFactory::ToHashInfo(HashFunction::Type hash_type, bazel_re::Digest const& digest) noexcept -> expected { bool const is_prefixed = IsPrefixed(hash_type, digest.hash()); auto hash = is_prefixed ? Unprefix(digest.hash()) : digest.hash(); auto const is_tree = is_prefixed and digest.hash().starts_with(kTreeTag); return HashInfo::Create(hash_type, std::move(hash), is_tree); } auto BazelDigestFactory::IsPrefixed(HashFunction::Type hash_type, std::string const& hash) noexcept -> bool { auto const tagged_length = HashFunction{hash_type}.MakeHasher().GetHashLength() + kTagLength; return hash.size() == tagged_length; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/bazel_digest_factory.hpp000066400000000000000000000063501516554100600305730ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_DIGEST_FACTORY_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_DIGEST_FACTORY_HPP #include #include #include #include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" class BazelDigestFactory final { static constexpr auto kBlobTag = "62"; static constexpr auto kTreeTag = "74"; static constexpr std::size_t kTagLength = 2; public: /// \brief Create bazel_re::Digest from preliminarily validated data. /// \param hash_data Validated hash /// \param size Size of the content [[nodiscard]] static auto Create(HashInfo const& hash_info, std::int64_t size) noexcept -> bazel_re::Digest; /// \brief Validate bazel_re::Digest /// \param hash_type Type of the hash function that was used for creation /// of the hash /// \param digest Digest to be validated /// \return Validated hash on success or an error message on failure. [[nodiscard]] static auto ToHashInfo( HashFunction::Type hash_type, bazel_re::Digest const& digest) noexcept -> expected; /// \brief Hash content using hash function and return a valid /// bazel_re::Digest /// \tparam kType Type of the hashing algorithm to be used /// \param hash_function Hash function to be used for hashing /// \param content Content to be hashed /// \return The digest of the content template [[nodiscard]] static auto HashDataAs(HashFunction hash_function, std::string const& content) -> bazel_re::Digest { auto const hash_info = HashInfo::HashData(hash_function, content, IsTreeObject(kType)); return Create(hash_info, gsl::narrow(content.size())); } private: [[nodiscard]] static auto Prefix(std::string const& hash, bool is_tree) noexcept -> std::string { return (is_tree ? kTreeTag : kBlobTag) + hash; } [[nodiscard]] static auto Unprefix(std::string const& hash) noexcept -> std::string { return hash.substr(kTagLength); } [[nodiscard]] static auto IsPrefixed(HashFunction::Type hash_type, std::string const& hash) noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_DIGEST_FACTORY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/bazel_types.hpp000066400000000000000000000061131516554100600267260ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP /// \file bazel_types.hpp /// \brief This file contains commonly used aliases for Bazel API /// Never include this file in any other header! #ifdef BOOTSTRAP_BUILD_TOOL #include #include #include namespace build::bazel::remote::execution::v2 { struct Digest { std::string hash_; std::int64_t size_bytes_; auto hash() const& noexcept -> std::string const& { return hash_; } auto size_bytes() const noexcept -> std::int64_t { return size_bytes_; } void set_size_bytes(std::int64_t size_bytes) { size_bytes_ = size_bytes; } void set_hash(std::string hash) { hash_ = hash; } std::string* mutable_hash() { return &hash_; } }; } // namespace build::bazel::remote::execution::v2 namespace google::protobuf { using int64 = std::int64_t; } #else #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.pb.h" #include "google/protobuf/message.h" #include "google/protobuf/repeated_ptr_field.h" #endif /// \brief Alias namespace for bazel remote execution // NOLINTNEXTLINE(misc-unused-alias-decls) namespace bazel_re = build::bazel::remote::execution::v2; #ifdef BOOTSTRAP_BUILD_TOOL // not using protobuffers #else /// \brief Alias namespace for 'google::protobuf' namespace pb { // NOLINTNEXTLINE(google-build-using-namespace) using namespace google::protobuf; /// \brief Alias function for 'RepeatedFieldBackInserter' template auto back_inserter(RepeatedField* const f) { return RepeatedFieldBackInserter(f); } /// \brief Alias function for 'RepeatedPtrFieldBackInserter' template auto back_inserter(RepeatedPtrField* const f) { return RepeatedPtrFieldBackInserter(f); } } // namespace pb #endif namespace std { /// \brief Hash function to support bazel_re::Digest as std::map* key. template <> struct hash { auto operator()(bazel_re::Digest const& d) const noexcept -> std::size_t { return std::hash{}(d.hash()); } }; /// \brief Equality function to support bazel_re::Digest as std::map* key. template <> struct equal_to { auto operator()(bazel_re::Digest const& lhs, bazel_re::Digest const& rhs) const noexcept -> bool { return lhs.hash() == rhs.hash(); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/cli.hpp000066400000000000000000001020651516554100600251570ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include #include "CLI/CLI.hpp" #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/symlinks/resolve_special.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/utils/cpp/path.hpp" inline constexpr auto kDefaultTimeout = std::chrono::milliseconds{300000}; inline constexpr auto kMaxOpCacheExponent = std::uint8_t{63}; /// \brief Arguments common to all commands. struct CommonArguments { std::optional workspace_root; std::optional repository_config; std::optional main; std::size_t jobs{std::max(1U, std::thread::hardware_concurrency())}; }; struct LogArguments { std::vector log_files; LogLevel log_limit{kDefaultLogLevel}; std::optional restrict_stderr_log_limit; bool plain_log{false}; bool log_append{false}; }; /// \brief Arguments required for analysing targets. struct AnalysisArguments { std::optional expression_log_limit; std::vector defines; std::filesystem::path config_file; std::optional target; std::optional request_action_input; std::optional target_file_name; std::optional rule_file_name; std::optional expression_file_name; std::optional target_root; std::optional rule_root; std::optional expression_root; std::vector graph_file; std::vector graph_file_plain; std::vector artifacts_to_build_files; std::optional serve_errors_file; std::optional profile; }; /// \brief Arguments required for describing targets/rules. struct DescribeArguments { bool print_json{}; bool describe_rule{}; }; /// \brief Arguments required for running diagnostics. struct DiagnosticArguments { std::optional dump_actions{std::nullopt}; std::optional dump_blobs{std::nullopt}; std::optional dump_trees{std::nullopt}; std::optional dump_provides{std::nullopt}; std::optional dump_vars{std::nullopt}; std::optional dump_targets{std::nullopt}; std::optional dump_export_targets{std::nullopt}; std::optional dump_targets_graph{std::nullopt}; std::optional dump_anonymous{std::nullopt}; std::optional dump_nodes{std::nullopt}; std::optional dump_result{std::nullopt}; }; /// \brief Arguments required for specifying build endpoint. struct EndpointArguments { std::optional local_root; std::optional remote_execution_address; std::string remote_instance_name{}; std::vector platform_properties; std::optional remote_execution_dispatch_file; }; /// \brief Arguments required for building. struct BuildArguments { std::optional> local_launcher{std::nullopt}; std::chrono::milliseconds timeout{kDefaultTimeout}; std::size_t build_jobs{}; std::vector dump_artifacts{}; std::optional print_to_stdout{std::nullopt}; bool print_unique{false}; bool show_runfiles{false}; }; /// \brief Arguments related to target-level caching struct TCArguments { TargetCacheWriteStrategy target_cache_write_strategy{ TargetCacheWriteStrategy::Sync}; }; /// \brief Arguments required for staging. struct StageArguments { std::filesystem::path output_dir; bool remember{false}; }; /// \brief Arguments required for rebuilding. struct RebuildArguments { std::optional cache_endpoint; std::optional dump_flaky; }; /// \brief Arguments for fetching artifacts from CAS. struct FetchArguments { std::string object_id; std::optional output_path; std::optional sub_path; bool remember{false}; bool raw_tree{}; bool archive{}; }; /// \brief Arguments required for running from graph file. struct GraphArguments { nlohmann::json artifacts; std::filesystem::path graph_file; std::optional git_cas; }; // Arguments for authentication methods. /// \brief Arguments shared by both server and client struct CommonAuthArguments { std::optional tls_ca_cert{std::nullopt}; }; /// \brief Arguments used by the client struct ClientAuthArguments { std::optional tls_client_cert{std::nullopt}; std::optional tls_client_key{std::nullopt}; }; /// \brief Authentication arguments used by subcommand just execute struct ServerAuthArguments { std::optional tls_server_cert{std::nullopt}; std::optional tls_server_key{std::nullopt}; }; struct ServiceArguments { std::optional port{std::nullopt}; std::optional info_file{std::nullopt}; std::optional interface{std::nullopt}; std::optional pid_file{std::nullopt}; std::optional op_exponent; }; struct ServeArguments { std::filesystem::path config; std::optional remote_serve_address; // repositories populated from just-serve config file std::vector repositories; // remote execution endpoint as seen by the client std::optional client_remote_address; }; struct GcArguments { bool no_rotate = false; bool all = false; }; struct ToAddArguments { std::filesystem::path location; bool follow_symlinks{}; std::optional resolve_special{std::nullopt}; }; struct ProtocolArguments final { HashFunction::Type hash_type = HashFunction::Type::GitSHA1; }; static inline auto SetupCommonArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("-C,--repository-config", clargs->repository_config, "Path to configuration file for multi-repository builds.") ->type_name("PATH"); app->add_option( "--main", clargs->main, "The repository to take the target from.") ->type_name("NAME"); app->add_option_function( "-w,--workspace-root", [clargs](auto const& workspace_root_raw) { std::filesystem::path root = ToNormalPath(workspace_root_raw); if (not root.is_absolute()) { try { root = std::filesystem::absolute(root); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Failed to convert workspace root {} ({})", workspace_root_raw, e.what()); throw e; } } clargs->workspace_root = root; }, "Path of the workspace's root directory.") ->type_name("PATH"); app->add_option("-j,--jobs", clargs->jobs, "Number of jobs to run (Default: Number of cores).") ->type_name("NUM"); } static inline auto SetupLogArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-f,--log-file", [clargs](auto const& log_file_) { clargs->log_files.emplace_back(log_file_); }, "Path to local log file.") ->type_name("PATH") ->trigger_on_parse(); // run callback on all instances while parsing, // not after all parsing is done app->add_option_function>( "--log-limit", [clargs](auto const& limit) { clargs->log_limit = ToLogLevel(limit); }, fmt::format("Log limit (higher is more verbose) in interval [{},{}] " "(Default: {}).", static_cast(kFirstLogLevel), static_cast(kLastLogLevel), static_cast(kDefaultLogLevel))) ->type_name("NUM"); app->add_option_function>( "--restrict-stderr-log-limit", [clargs](auto const& limit) { clargs->restrict_stderr_log_limit = ToLogLevel(limit); }, "Restrict logging on console to the minimum of the specified " "--log-limit and this value") ->type_name("NUM"); app->add_flag("--plain-log", clargs->plain_log, "Do not use ANSI escape sequences to highlight messages."); app->add_flag( "--log-append", clargs->log_append, "Append messages to log file instead of overwriting existing."); } static inline auto SetupAnalysisArguments( gsl::not_null const& app, gsl::not_null const& clargs, bool with_graph = true) { app->add_option("--expression-log-limit", clargs->expression_log_limit, fmt::format("Maximal size for logging a single expression " "in error messages (Default {})", Evaluator::kDefaultExpressionLogLimit)) ->type_name("NUM"); app->add_option_function( "-D,--defines", [clargs](auto const& d) { clargs->defines.emplace_back(d); }, "Define an overlay configuration via an in-line JSON object." " Multiple options overlay.") ->type_name("JSON") ->trigger_on_parse(); // run callback on all instances while parsing, // not after all parsing is done app->add_option( "-c,--config", clargs->config_file, "Path to configuration file.") ->type_name("PATH"); app->add_option( "--request-action-input", clargs->request_action_input, "Instead of the target result, request input for this action.") ->type_name("ACTION"); app->add_option_function>( "target", [clargs](auto const& target_raw) { if (target_raw.size() == 1) { clargs->target = nlohmann::json(target_raw[0]); } else { clargs->target = nlohmann::json(target_raw); } }, "Module and target name to build.\n" "Assumes current module if module name is omitted."); app->add_option("--target-root", clargs->target_root, "Path of the target files' root directory.\n" "Default: Same as --workspace-root") ->type_name("PATH"); app->add_option("--rule-root", clargs->rule_root, "Path of the rule files' root directory.\n" "Default: Same as --target-root") ->type_name("PATH"); app->add_option("--expression-root", clargs->expression_root, "Path of the expression files' root directory.\n" "Default: Same as --rule-root") ->type_name("PATH"); app->add_option("--target-file-name", clargs->target_file_name, "Name of the targets file."); app->add_option( "--rule-file-name", clargs->rule_file_name, "Name of the rules file."); app->add_option("--expression-file-name", clargs->expression_file_name, "Name of the expressions file."); app->add_option("--serve-errors-log", clargs->serve_errors_file, "File path for dumping the blob identifiers of serve " "errors as json.") ->type_name("PATH"); app->add_option( "--profile", clargs->profile, "Location to write the profile to.") ->type_name("PATH"); if (with_graph) { app->add_option_function( "--dump-graph", [clargs](auto const& file_) { clargs->graph_file.emplace_back(file_); }, "File path for writing the action graph description to.") ->type_name("PATH") ->trigger_on_parse(); app->add_option_function( "--dump-plain-graph", [clargs](auto const& file_) { clargs->graph_file_plain.emplace_back(file_); }, "File path for writing the action graph description to.") ->type_name("PATH") ->trigger_on_parse(); app->add_option_function( "--dump-artifacts-to-build", [clargs](auto const& file_) { clargs->artifacts_to_build_files.emplace_back(file_); }, "File path for writing the artifacts to build to.") ->type_name("PATH") ->trigger_on_parse(); } } static inline auto SetupDescribeArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_flag("--json", clargs->print_json, "Omit pretty-printing and describe rule in JSON format."); app->add_flag("--rule", clargs->describe_rule, "Positional arguments refer to rule instead of target."); } static inline auto SetupDiagnosticArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("--dump-actions", clargs->dump_actions, "Dump actions to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-trees", clargs->dump_trees, "Dump trees to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-blobs", clargs->dump_blobs, "Dump blobs to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-provides", clargs->dump_provides, "Dump provides map to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-vars", clargs->dump_vars, "Dump domain of the effective configuration to file (use - " "for stdout).") ->type_name("PATH"); app->add_option("--dump-targets", clargs->dump_targets, "Dump targets to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-export-targets", clargs->dump_export_targets, "Dump \"export\" targets to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-targets-graph", clargs->dump_targets_graph, "Dump the graph of the configured targets to file.") ->type_name("PATH"); app->add_option("--dump-anonymous", clargs->dump_anonymous, "Dump anonymous targets to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-nodes", clargs->dump_nodes, "Dump nodes of target to file (use - for stdout).") ->type_name("PATH"); app->add_option("--dump-result", clargs->dump_result, "Dump the result of analyse to file (use - for stdout).") ->type_name("PATH"); } static inline auto SetupCacheArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "--local-build-root", [clargs](auto const& build_root_raw) { std::filesystem::path root = ToNormalPath(build_root_raw); if (not root.is_absolute()) { try { root = std::filesystem::absolute(root); } catch (std::exception const& e) { Logger::Log( LogLevel::Error, "Failed to convert local build root {} ({}).", build_root_raw, e.what()); throw e; } } clargs->local_root = root; }, "Root for local CAS, cache, and build directories.") ->type_name("PATH"); } static inline auto SetupExecutionEndpointArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("-r,--remote-execution-address", clargs->remote_execution_address, "Address of the remote-execution service.") ->type_name("NAME:PORT"); app->add_option( "--remote-instance-name", clargs->remote_instance_name, "Instance name of the remote-execution service (default: \"\")") ->type_name("STRING"); } static inline auto SetupExecutionPropertiesArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("--endpoint-configuration", clargs->remote_execution_dispatch_file, "File with dispatch instructions to use different " "remote-execution services, depending on the properties") ->type_name("PATH"); app->add_option( "--remote-execution-property", clargs->platform_properties, "Property for remote execution as key-value pair. Specifying this " "option multiple times will accumulate pairs (latest wins).") ->type_name("KEY:VAL") ->allow_extra_args(false) ->expected(1) ->take_all(); } static inline auto SetupServeEndpointArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("-R,--remote-serve-address", clargs->remote_serve_address, "Address of the serve service.") ->type_name("NAME:PORT"); } static inline auto SetupCommonBuildArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-L,--local-launcher", [clargs](auto const& launcher_raw) { clargs->local_launcher = nlohmann::json::parse(launcher_raw) .template get>(); }, "JSON array with the list of strings representing the launcher to " "prepend actions' commands before being executed locally.") ->type_name("JSON") ->default_val(nlohmann::json(kDefaultLauncher).dump()); } static inline auto SetupBuildArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "--action-timeout", [clargs](auto const& seconds) { clargs->timeout = seconds * std::chrono::seconds{1}; }, "Action timeout in seconds. (Default: 300).") ->type_name("NUM"); app->add_option( "-J,--build-jobs", clargs->build_jobs, "Number of jobs to run during build phase (Default: same as jobs).") ->type_name("NUM"); } static inline auto SetupExtendedBuildArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "--dump-artifacts", [clargs](auto const& file_) { clargs->dump_artifacts.emplace_back(file_); }, "Dump artifacts to file (use - for stdout).") ->type_name("PATH") ->trigger_on_parse(); app->add_flag("-s,--show-runfiles", clargs->show_runfiles, "Do not omit runfiles in build report."); app->add_option("-P,--print-to-stdout", clargs->print_to_stdout, "After building, print the specified artifact to stdout.") ->type_name("LOGICAL_PATH"); app->add_flag("-p,--print-unique-artifact", clargs->print_unique, "Print the unique artifact, if any, to stdout."); } static inline auto SetupTCArguments(gsl::not_null const& app, gsl::not_null const& tcargs) { app->add_option_function( "--target-cache-write-strategy", [tcargs](auto const& s) { auto strategy = ToTargetCacheWriteStrategy(s); if (strategy) { tcargs->target_cache_write_strategy = *strategy; } else { Logger::Log(LogLevel::Warning, "Ignoring unknown strategy {} to write " "target-level cache.", nlohmann::json(s).dump()); } }, "Strategy for writing target-cache. (Default: sync)") ->type_name("STRATEGY"); } static inline auto SetupStageArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-o,--output-dir", [clargs](auto const& output_dir_raw) { std::filesystem::path out = ToNormalPath(output_dir_raw); if (not out.is_absolute()) { try { out = std::filesystem::absolute(out); } catch (std::exception const& e) { Logger::Log( LogLevel::Error, "Failed to convert output directory {} ({}).", output_dir_raw, e.what()); throw e; } } clargs->output_dir = out; }, "Path of the directory where outputs will be copied.") ->type_name("PATH") ->required(); app->add_flag( "--remember", clargs->remember, "Copy object to local CAS first"); } static inline auto SetupRebuildArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "--vs", [clargs](auto const& cache_endpoint) { clargs->cache_endpoint = cache_endpoint; }, "Cache endpoint to compare against (use \"local\" for local cache).") ->type_name("NAME:PORT|\"local\""); app->add_option( "--dump-flaky", clargs->dump_flaky, "Dump flaky actions to file.") ->type_name("PATH"); } static inline auto SetupFetchArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option( "object_id", clargs->object_id, "Object identifier with the format '[::]'.") ->required(); app->add_option_function( "-o,--output-path", [clargs](auto const& output_path_raw) { std::filesystem::path out = ToNormalPath(output_path_raw); if (not out.is_absolute()) { try { out = std::filesystem::absolute(out); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Failed to convert output path {} ({})", output_path_raw, e.what()); throw e; } } clargs->output_path = out; }, "Install path for the artifact. (omit to dump to stdout)") ->type_name("PATH"); app->add_option_function( "-P,--sub-object-path", [clargs](auto const& rel_path) { clargs->sub_path = ToNormalPath(rel_path).relative_path(); }, "Select the sub-object at the specified path (if artifact is a " "tree).") ->type_name("PATH"); app->add_flag( "--archive", clargs->archive, "Dump the tree as a single archive."); app->add_flag("--raw-tree", clargs->raw_tree, "Dump raw tree object (omit pretty printing)."); app->add_flag( "--remember", clargs->remember, "Copy object to local CAS first"); } static inline auto SetupToAddArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "location", [clargs](auto const& path_raw) { std::filesystem::path in = ToNormalPath(path_raw); if (not in.is_absolute()) { try { in = std::filesystem::absolute(in); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Failed to convert input path {} ({})", path_raw, e.what()); throw e; } } clargs->location = in; }, "The path on the local file system to be added to CAS") ->required(); app->add_flag("--follow-symlinks", clargs->follow_symlinks, "Resolve the positional argument to not be a symbolic link " "before adding it to CAS."); app->add_option_function( "--resolve-special", [clargs](auto const& raw_value) { if (auto const it = kResolveSpecialMap.find(raw_value); it != kResolveSpecialMap.end()) { clargs->resolve_special = it->second; } else { Logger::Log(LogLevel::Warning, "Ignoring unknown --resolve-special strategy {}.", nlohmann::json(raw_value).dump()); } }, "Optional strategy for handling special entries (e.g., symlinks) when " "the content to add is a directory. Default behaviour if missing is to " "only allow non-upward symlinks. Currently supported values: ignore, " "tree-upwards, tree-all, all."); } static inline auto SetupGraphArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-a,--artifacts", [clargs](auto const& artifact_map_raw) { clargs->artifacts = nlohmann::json::parse(artifact_map_raw); }, "Json object with key/value pairs formed by the relative path in which " "artifact is to be copied and the description of the artifact as json " "object as well."); app->add_option("-g,--graph-file", clargs->graph_file, "Path of the file containing the description of the " "actions.") ->required(); app->add_option("--git-cas", clargs->git_cas, "Path to a Git repository, containing blobs of potentially " "missing KNOWN artifacts."); } static inline auto SetupProtocolArguments( gsl::not_null const& app, gsl::not_null const& protocol) { app->add_flag_function( "--compatible", [protocol](auto /*unused*/) { protocol->hash_type = HashFunction::Type::PlainSHA256; }, "At increased computational effort, be compatible with the original " "remote build execution protocol. As the change affects identifiers, " "the flag must be used consistently for all related invocations."); } static inline auto SetupCommonAuthArguments( gsl::not_null const& app, gsl::not_null const& authargs) { app->add_option("--tls-ca-cert", authargs->tls_ca_cert, "Path to a TLS CA certificate that is trusted to sign the " "server certificate."); } static inline auto SetupClientAuthArguments( gsl::not_null const& app, gsl::not_null const& authargs) { app->add_option("--tls-client-cert", authargs->tls_client_cert, "Path to the TLS client certificate."); app->add_option("--tls-client-key", authargs->tls_client_key, "Path to the TLS client key."); } static inline auto SetupServerAuthArguments( gsl::not_null const& app, gsl::not_null const& authargs) { app->add_option("--tls-server-cert", authargs->tls_server_cert, "Path to the TLS server certificate."); app->add_option("--tls-server-key", authargs->tls_server_key, "Path to the TLS server key."); } static inline auto SetupServiceArguments( gsl::not_null const& app, gsl::not_null const& service_args) { app->add_option("-p,--port", service_args->port, "The service will listen to this port. If unset, the " "service will listen to the first available one."); app->add_option("--info-file", service_args->info_file, "Write the used port, interface, and pid to this file in " "JSON format. If the file exists, it " "will be overwritten."); app->add_option("-i,--interface", service_args->interface, "Interface to use. If unset, the loopback device is used."); app->add_option( "--pid-file", service_args->pid_file, "Write pid to this file in plain txt. If the file exists, it " "will be overwritten."); app->add_option_function( "--log-operations-threshold", [service_args](auto const& op_exponent) { if (op_exponent > kMaxOpCacheExponent) { Logger::Log(LogLevel::Warning, "Ignoring invalid value {} for operations " "threshold exponent.", op_exponent); } else { service_args->op_exponent = op_exponent; } }, "Once the number of operations stored exceeds twice 2^n, where n is " "given by the option --log-operations-threshold, at most 2^n " "operations will be removed, in a FIFO scheme. If unset, defaults to " "14. Must be in the range [0,63]"); } static inline auto SetupServeArguments( gsl::not_null const& app, gsl::not_null const& serve_args) { app->add_option("config", serve_args->config, "Configuration file for the subcommand.") ->required(); } static inline void SetupGcArguments(gsl::not_null const& app, gsl::not_null const& args) { auto* no_rotate = app->add_flag("--no-rotate", args->no_rotate, "Do not rotate cache generations, only clean up what can " "be done without losing cache."); auto* all = app->add_flag( "--all", args->all, "Remove all cache generations at once"); no_rotate->excludes(all); all->excludes(no_rotate); } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/clidefaults.hpp000066400000000000000000000020121516554100600266760ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_CLIDEFAULTS_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_CLIDEFAULTS_HPP #include #include #include "src/buildtool/logging/log_level.hpp" constexpr auto kDefaultLogLevel = LogLevel::Progress; static inline const std::vector kDefaultLauncher = std::vector{"env", "--"}; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLIDEFAULTS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/git_hashes_converter.hpp000066400000000000000000000061051516554100600306130ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_GIT_HASHES_CONVERTER_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_GIT_HASHES_CONVERTER_HPP #include #include //std::unique_lock #include #include #include #include #include #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" class GitHashesConverter final { using git_hash = std::string; using compat_hash = std::string; using git_repo = std::string; using GitToCompatibleMap = std::unordered_map; using CompatibleToGitMap = std::unordered_map>; public: [[nodiscard]] static auto Instance() noexcept -> GitHashesConverter& { static GitHashesConverter instance; return instance; } [[nodiscard]] auto RegisterGitEntry(std::string const& git_hash, std::string const& data, std::string const& repo) -> compat_hash { { std::shared_lock lock{mutex_}; auto it = git_to_compatible_.find(git_hash); if (it != git_to_compatible_.end()) { return it->second; } } // This is only used in compatible mode. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; auto compatible_hash = hash_function.PlainHashData(data).HexString(); std::unique_lock lock{mutex_}; git_to_compatible_[git_hash] = compatible_hash; compatible_to_git_[compatible_hash] = {git_hash, repo}; return compatible_hash; } [[nodiscard]] auto GetGitEntry(std::string const& compatible_hash) -> std::optional> { std::shared_lock lock{mutex_}; auto it = compatible_to_git_.find(compatible_hash); if (it != compatible_to_git_.end()) { return it->second; } Logger::Log(LogLevel::Warning, "Unable to get the git-sha1 code associated to {}", compatible_hash); return std::nullopt; } private: explicit GitHashesConverter() noexcept = default; GitToCompatibleMap git_to_compatible_; CompatibleToGitMap compatible_to_git_; std::shared_mutex mutex_; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_GIT_HASHES_CONVERTER_HPPjust-buildsystem-justbuild-b1fb5fa/src/buildtool/common/identifier.hpp000066400000000000000000000026411516554100600265310ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP #include #include #include #include // Global artifact identifier (not the CAS hash) using ArtifactIdentifier = std::string; // Global action identifier using ActionIdentifier = std::string; static inline auto IdentifierToString(const std::string& id) -> std::string { std::ostringstream encoded{}; encoded << std::hex << std::setfill('0'); for (auto const& b : id) { encoded << std::setw(2) << static_cast( static_cast>>(b)); } return encoded.str(); } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/location.cpp000066400000000000000000000042651516554100600262160ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/location.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" auto ReadLocationObject(nlohmann::json const& location, std::optional const& ws_root) -> expected, std::string> { if (not location.contains("path") or not location.contains("root")) { return unexpected( fmt::format("Malformed location object: {}", location.dump(-1))); } auto root = location["root"].get(); auto path = location["path"].get(); auto base = location.contains("base") ? location["base"].get() : std::string("."); std::filesystem::path root_path{}; if (root == "workspace") { if (not ws_root) { Logger::Log(LogLevel::Warning, "Not in workspace root, ignoring location {}.", location.dump(-1)); return std::optional{std::nullopt}; } root_path = *ws_root; } if (root == "home") { root_path = FileSystemManager::GetUserHome(); } if (root == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); } return std::optional{ std::make_pair(std::filesystem::weakly_canonical(root_path / path), std::filesystem::weakly_canonical(root_path / base))}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/location.hpp000066400000000000000000000023131516554100600262130ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include "nlohmann/json.hpp" #include "src/utils/cpp/expected.hpp" using location_res_t = std::pair; /// \brief Parse a location object stored in a JSON object. /// \returns optional parsed location (nullopt if location should be ignored) on /// success or unexpected error as string. [[nodiscard]] auto ReadLocationObject( nlohmann::json const& location, std::optional const& ws_root) -> expected, std::string>; just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/protocol_traits.hpp000066400000000000000000000022431516554100600276340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_PROTOCOL_TRAITS_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_PROTOCOL_TRAITS_HPP #include "src/buildtool/crypto/hash_function.hpp" class ProtocolTraits final { public: static constexpr auto IsNative(HashFunction::Type hash_type) noexcept -> bool { return hash_type == HashFunction::Type::GitSHA1; } static constexpr auto IsTreeAllowed(HashFunction::Type hash_type) noexcept -> bool { return IsNative(hash_type); } }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_PROTOCOL_TRAITS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/000077500000000000000000000000001516554100600251665ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/TARGETS000066400000000000000000000035411516554100600262250ustar00rootroot00000000000000{ "client_common": { "type": ["@", "rules", "CC", "library"] , "name": ["client_common"] , "hdrs": ["client_common.hpp"] , "proto": [["@", "googleapis", "", "google_rpc_status_proto"]] , "deps": [ "port" , ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "common", "remote"] } , "port": { "type": ["@", "rules", "CC", "library"] , "name": ["port"] , "hdrs": ["port.hpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "type_safe_arithmetic"] ] , "stage": ["src", "buildtool", "common", "remote"] } , "remote_common": { "type": ["@", "rules", "CC", "library"] , "name": ["remote_common"] , "hdrs": ["remote_common.hpp"] , "deps": [ "port" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "common", "remote"] } , "retry_config": { "type": ["@", "rules", "CC", "library"] , "name": ["retry_config"] , "hdrs": ["retry_config.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "common", "remote"] } , "retry": { "type": ["@", "rules", "CC", "library"] , "name": ["retry"] , "hdrs": ["retry.hpp"] , "srcs": ["retry.cpp"] , "deps": [ "retry_config" , ["@", "grpc", "", "grpc++"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "common", "remote"] , "private-deps": [["@", "fmt", "", "fmt"], ["@", "gsl", "", "gsl"]] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/client_common.hpp000066400000000000000000000061561516554100600305350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_CLIENT_COMMON_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_CLIENT_COMMON_HPP /// \file client_common.hpp /// \brief Common types and functions required by client implementations. #include #include #include #include #include #include "fmt/core.h" #include "google/rpc/status.pb.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" [[maybe_unused]] [[nodiscard]] static inline auto CreateChannelWithCredentials( std::string const& server, Port port, gsl::not_null const& auth) noexcept { std::shared_ptr creds; std::string address = server + ':' + std::to_string(port); if (const auto* tls_auth = std::get_if(&auth->method); tls_auth != nullptr) { auto tls_opts = grpc::SslCredentialsOptions{ tls_auth->ca_cert, tls_auth->client_key, tls_auth->client_cert}; creds = grpc::SslCredentials(tls_opts); } else { // currently only TLS/SSL is supported creds = grpc::InsecureChannelCredentials(); } return grpc::CreateChannel(address, creds); } [[nodiscard]] static inline auto StatusString( grpc::Status const& s, std::optional const& prefix = std::nullopt) noexcept -> std::string { return fmt::format("{}{}: {}", (prefix.has_value() ? fmt::format("{}: ", *prefix) : ""), static_cast(s.error_code()), s.error_message()); } [[nodiscard]] static inline auto StatusString( google::rpc::Status const& s, std::optional const& prefix = std::nullopt) noexcept -> std::string { return fmt::format("{}{}: {}", (prefix.has_value() ? fmt::format("{}: ", *prefix) : ""), static_cast(s.code()), s.message()); } template [[maybe_unused]] static inline void LogStatus( Logger const* logger, LogLevel level, TStatus const& s, std::optional const& prefix = std::nullopt) noexcept { auto msg = [&s, &prefix]() { return StatusString(s, prefix); }; if (logger == nullptr) { Logger::Log(level, msg); } else { logger->Emit(level, msg); } } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLIENT_COMMON_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/port.hpp000066400000000000000000000037021516554100600266650ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_PORT_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_PORT_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/type_safe_arithmetic.hpp" // Port struct PortTag : TypeSafeArithmeticTag {}; using Port = TypeSafeArithmetic; [[nodiscard]] static inline auto ParsePort(int const port_num) noexcept -> std::optional { try { static constexpr int kMaxPortNumber{ std::numeric_limits::max()}; if (port_num >= 0 and port_num <= kMaxPortNumber) { return gsl::narrow_cast(port_num); } } catch (std::out_of_range const& e) { Logger::Log(LogLevel::Error, "Port raised out_of_range exception."); } return std::nullopt; } [[nodiscard]] static inline auto ParsePort(std::string const& port) noexcept -> std::optional { try { auto port_num = std::stoi(port); return ParsePort(port_num); } catch (std::invalid_argument const& e) { Logger::Log(LogLevel::Error, "Port raised invalid_argument exception."); } return std::nullopt; } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_PORT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/remote_common.hpp000066400000000000000000000116771516554100600305560ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_REMOTE_COMMON_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_REMOTE_COMMON_HPP #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/utils/cpp/expected.hpp" struct ServerAddress { std::string host; Port port; [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json { return nlohmann::json( fmt::format("{}:{}", host, static_cast(port))); } }; // Useful additional types using ExecutionProperties = std::map; using DispatchEndpoint = std::pair; [[nodiscard]] static inline auto ParseAddress( std::string const& address) noexcept -> std::optional { std::istringstream iss(address); std::string host; std::string port; if (not std::getline(iss, host, ':') or not std::getline(iss, port, ':')) { return std::nullopt; } auto port_num = ParsePort(port); if (not host.empty() and port_num) { return ServerAddress{std::move(host), *port_num}; } return std::nullopt; } [[nodiscard]] static inline auto ParseProperty( std::string const& property) noexcept -> std::optional> { std::istringstream pss(property); std::string key; std::string val; if (not std::getline(pss, key, ':') or not std::getline(pss, val)) { return std::nullopt; } return std::make_pair(key, val); } [[nodiscard]] static inline auto ParseDispatch( std::string const& dispatch_info) noexcept -> expected, std::string> { nlohmann::json dispatch; try { dispatch = nlohmann::json::parse(dispatch_info); } catch (std::exception const& e) { return unexpected{fmt::format( "Failed to parse endpoint configuration: {}", e.what())}; } std::vector parsed{}; try { if (not dispatch.is_array()) { return unexpected{ fmt::format("Endpoint configuration has to be a " "list of pairs, but found {}", dispatch.dump())}; } for (auto const& entry : dispatch) { if (not entry.is_array() or entry.size() != 2) { return unexpected{ fmt::format("Endpoint configuration has to be a list of " "pairs, but found entry {}", entry.dump())}; } if (not entry[0].is_object()) { return unexpected{ fmt::format("Property condition has to be " "given as an object, but found {}", entry[0].dump())}; } ExecutionProperties props{}; for (auto const& [k, v] : entry[0].items()) { if (not v.is_string()) { return unexpected{ fmt::format("Property condition has to be given as an " "object of strings but found {}", entry[0].dump())}; } props.emplace(k, v.template get()); } if (not entry[1].is_string()) { return unexpected{ fmt::format("Endpoint has to be specified as string (in " "the form host:port), but found {}", entry[1].dump())}; } auto endpoint = ParseAddress(entry[1].template get()); if (not endpoint) { return unexpected{fmt::format("Failed to parse {} as endpoint.", entry[1].dump())}; } parsed.emplace_back(props, *endpoint); } } catch (std::exception const& e) { return unexpected{ fmt::format("Failure analysing endpoint configuration {}: {}", dispatch.dump(), e.what())}; } // success! return parsed; } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_REMOTE_COMMON_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/retry.cpp000066400000000000000000000200451516554100600270400ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/remote/retry.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include "fmt/core.h" #include "gsl/gsl" auto WithRetry(CallableReturningRetryResponse const& f, RetryConfig const& retry_config, Logger const& logger, LogLevel fatal_log_level) noexcept -> bool { try { auto const& attempts = retry_config.GetMaxAttempts(); for (auto attempt = 1U; attempt <= attempts; ++attempt) { auto [ok, fatal, error_msg] = f(); if (ok) { return true; } if (fatal) { if (error_msg) { logger.Emit(fatal_log_level, "{}", *error_msg); } return false; } // don't wait if it was the last attempt if (attempt < attempts) { auto const sleep_for_seconds = retry_config.GetSleepTimeSeconds(attempt); logger.Emit(kRetryLogLevel, "Attempt {}/{} failed{} Retrying in {} seconds.", attempt, attempts, error_msg ? fmt::format(": {}", *error_msg) : ".", sleep_for_seconds); std::this_thread::sleep_for( std::chrono::seconds(sleep_for_seconds)); } else { if (error_msg) { logger.Emit(fatal_log_level, "After {} attempts: {}", attempt, *error_msg); } } } } catch (...) { logger.Emit(std::min(fatal_log_level, LogLevel::Warning), "WithRetry: caught unknown exception"); } return false; } auto WithRetry(CallableReturningGrpcStatus const& f, RetryConfig const& retry_config, Logger const& logger) noexcept -> std::pair { grpc::Status status{}; try { auto attempts = retry_config.GetMaxAttempts(); for (auto attempt = 1U; attempt <= attempts; ++attempt) { status = f(); if (status.ok() or ((status.error_code() != grpc::StatusCode::UNAVAILABLE) and (status.error_code() != grpc::StatusCode::DEADLINE_EXCEEDED))) { return {status.ok(), std::move(status)}; } // don't wait if it was the last attempt if (attempt < attempts) { auto const sleep_for_seconds = retry_config.GetSleepTimeSeconds(attempt); logger.Emit( kRetryLogLevel, "Attempt {}/{} failed: {}: {}: Retrying in {} seconds.", attempt, attempts, static_cast(status.error_code()), status.error_message(), sleep_for_seconds); std::this_thread::sleep_for( std::chrono::seconds(sleep_for_seconds)); } else { // The caller performs a second check on the // status.error_code(), and, eventually, emits to Error level // there. // // To avoid duplication of similar errors, we emit to Debug // level. logger.Emit(LogLevel::Debug, "After {} attempts: {}: {}", attempt, static_cast(status.error_code()), status.error_message()); } } } catch (...) { logger.Emit(LogLevel::Error, "WithRetry: caught unknown exception"); } return {false, std::move(status)}; } [[nodiscard]] auto IsReasonableToRetry(grpc::Status const& status) noexcept -> bool { // NOLINTBEGIN(bugprone-branch-clone,-warnings-as-errors) switch (status.error_code()) { case grpc::StatusCode::OK: { // Success, don't retry return false; } case grpc::StatusCode::CANCELLED: { // Operation canceled by the user, don't retry return false; } case grpc::StatusCode::UNKNOWN: { // Errors raised by APIs that do not return enough error information // may be converted to this error, don't retry return false; } case grpc::StatusCode::INVALID_ARGUMENT: { // Client specified an invalid argument, don't retry return false; } case grpc::StatusCode::DEADLINE_EXCEEDED: { // Deadline expired before operation could complete, retry return true; } case grpc::StatusCode::NOT_FOUND: { // Requested entity was not found, don't retry return false; } case grpc::StatusCode::ALREADY_EXISTS: { // Entity that we attempted to create (e.g., file or directory) // already exists, don't retry return false; } case grpc::StatusCode::PERMISSION_DENIED: { // The caller does not have permission to execute the specified // operation, don't retry return false; } case grpc::StatusCode::UNAUTHENTICATED: { // The request does not have valid authentication credentials, don't // retry return false; } case grpc::StatusCode::RESOURCE_EXHAUSTED: { // Some resource has been exhausted, perhaps a per-user quota, or // perhaps the entire file system is out of space, retry: return true; } case grpc::StatusCode::FAILED_PRECONDITION: { // Client performs conditional REST operation on a resource and the // resource on the server does not match the condition, don't retry return false; } case grpc::StatusCode::ABORTED: { // Client should retry at a higher-level, don't retry return false; } case grpc::StatusCode::OUT_OF_RANGE: { // Operation was attempted past the valid range. E.g., seeking or // reading past end of file. Unlike INVALID_ARGUMENT, this error // indicates a problem that may be fixed if the system state // changes. return true; } case grpc::StatusCode::UNIMPLEMENTED: { // Operation is not implemented or not supported/enabled in this // service, don't retry return false; } case grpc::StatusCode::INTERNAL: { // Something is very broken, don't retry return false; } case grpc::StatusCode::UNAVAILABLE: { // The service is currently unavailable, retry: return true; } case grpc::StatusCode::DATA_LOSS: { // Unrecoverable data loss or corruption, retry: return true; } case grpc::StatusCode::DO_NOT_USE: { // "Force users to include a default branch", but we want a compile // error if a new case appears, so check DO_NOT_USE explicitly return false; } } // NOLINTEND(bugprone-branch-clone,-warnings-as-errors) Expects(false); // unreachable } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/retry.hpp000066400000000000000000000060151516554100600270460ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" // Utility class to help detecting when exit the retry loop. This class can be // used when the failure cannot be immediately detected by the return value of // the function. E.g., when using a grpc stream. // // Please note that it is user's responsibility to do not set both to true. // // Design note: even though only one bool could be sufficient (e.g., exit), this // would require to check two times if we exited because of a success or a // failure: the first time, inside the retry loop; the second time, by the // caller. struct RetryResponse { // When set to true, it means the function successfully run bool ok = false; // When set to true, it means that it is not worthy to retry. bool exit_retry_loop = false; // error message logged when exit_retry_loop was set to true or when the // last retry attempt failed std::optional error_msg = std::nullopt; }; using CallableReturningRetryResponse = std::function; /// \brief Calls a function with a retry strategy using a backoff algorithm. /// Retry loop interrupts when one of the two members of the function's returned /// RetryResponse object is set to true. [[nodiscard]] auto WithRetry( CallableReturningRetryResponse const& f, RetryConfig const& retry_config, Logger const& logger, LogLevel fatal_log_level = LogLevel::Error) noexcept -> bool; using CallableReturningGrpcStatus = std::function; /// \brief Calls a function with a retry strategy using a backoff algorithm. /// Retry loop interrupts when function returns an error code different from /// UNAVAILABLE. [[nodiscard]] auto WithRetry(CallableReturningGrpcStatus const& f, RetryConfig const& retry_config, Logger const& logger) noexcept -> std::pair; /// \brief Check if it is reasonable to retry. [[nodiscard]] auto IsReasonableToRetry(grpc::Status const& status) noexcept -> bool; #endif // BOOTSTRAP_BUILD_TOOL #endif // INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/remote/retry_config.hpp000066400000000000000000000125701516554100600303760ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_PARAMETERS_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_PARAMETERS_HPP #include #include #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/expected.hpp" inline constexpr unsigned int kDefaultInitialBackoffSeconds{1}; inline constexpr unsigned int kDefaultMaxBackoffSeconds{60}; inline constexpr unsigned int kDefaultAttempts{1}; inline constexpr auto kRetryLogLevel = LogLevel::Progress; class RetryConfig final { public: class Builder; RetryConfig() = default; [[nodiscard]] auto GetMaxAttempts() const noexcept -> unsigned int { return attempts_; } /// \brief The waiting time is exponentially increased at each \p /// attempt until it exceeds max_backoff_seconds. /// /// To avoid overloading of the reachable resources, a jitter (aka, /// random value) is added to distribute the workload. [[nodiscard]] auto GetSleepTimeSeconds(unsigned int attempt) const noexcept -> unsigned int { auto backoff = initial_backoff_seconds_; // on the first attempt, we don't double the backoff time // also we do it in a for loop to avoid overflow for (auto x = 1U; x < attempt; ++x) { backoff <<= 1U; if (backoff >= max_backoff_seconds_) { backoff = max_backoff_seconds_; break; } } return backoff + Jitter(backoff); } private: unsigned int const initial_backoff_seconds_ = kDefaultInitialBackoffSeconds; unsigned int const max_backoff_seconds_ = kDefaultMaxBackoffSeconds; unsigned int const attempts_ = kDefaultAttempts; RetryConfig(unsigned int initial_backoff_seconds, unsigned int max_backoff_seconds, unsigned int attempts) : initial_backoff_seconds_{initial_backoff_seconds}, max_backoff_seconds_{max_backoff_seconds}, attempts_{attempts} {} using dist_type = std::uniform_int_distribution; [[nodiscard]] static auto Jitter(unsigned int backoff) noexcept -> typename dist_type::result_type { static std::mutex mutex; static std::mt19937 rng{std::random_device{}()}; try { dist_type dist{0, 3UL * backoff}; std::unique_lock lock(mutex); return dist(rng); } catch (...) { return 0; } } }; class RetryConfig::Builder final { public: auto SetInitialBackoffSeconds(std::optional x) noexcept -> Builder& { initial_backoff_seconds_ = x; return *this; } auto SetMaxBackoffSeconds(std::optional x) noexcept -> Builder& { max_backoff_seconds_ = x; return *this; } auto SetMaxAttempts(std::optional x) noexcept -> Builder& { attempts_ = x; return *this; } [[nodiscard]] auto Build() const noexcept -> expected { unsigned int initial_backoff_seconds = kDefaultInitialBackoffSeconds; if (initial_backoff_seconds_.has_value()) { if (*initial_backoff_seconds_ < 1) { return unexpected{ fmt::format("Invalid initial amount of seconds provided: " "{}.\nValue must be strictly greater than 0.", *initial_backoff_seconds_)}; } initial_backoff_seconds = *initial_backoff_seconds_; } unsigned int max_backoff_seconds = kDefaultMaxBackoffSeconds; if (max_backoff_seconds_.has_value()) { if (*max_backoff_seconds_ < 1) { return unexpected{ fmt::format("Invalid max backoff provided: {}.\nValue must " "be strictly greater than 0.", *max_backoff_seconds_)}; } max_backoff_seconds = *max_backoff_seconds_; } unsigned int attempts = kDefaultAttempts; if (attempts_.has_value()) { if (*attempts_ < 1) { return unexpected{ fmt::format("Invalid max number of attempts provided: " "{}.\nValue must be strictly greater than 0.", *attempts_)}; } attempts = *attempts_; } return RetryConfig( initial_backoff_seconds, max_backoff_seconds, attempts); } private: std::optional initial_backoff_seconds_; std::optional max_backoff_seconds_; std::optional attempts_; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_PARAMETERS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/repository_config.cpp000066400000000000000000000174711516554100600301550ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/repository_config.hpp" #include #include "src/buildtool/file_system/git_tree_utils.hpp" #include "src/utils/automata/dfa_minimizer.hpp" auto RepositoryConfig::RepositoryInfo::BaseContentDescription() const -> std::optional { auto wroot = workspace_root.ContentDescription(); auto troot = target_root.ContentDescription(); auto rroot = rule_root.ContentDescription(); auto eroot = expression_root.ContentDescription(); if (wroot and troot and rroot and eroot) { return nlohmann::json{{"workspace_root", *wroot}, {"target_root", *troot}, {"rule_root", *rroot}, {"expression_root", *eroot}, {"target_file_name", target_file_name}, {"rule_file_name", rule_file_name}, {"expression_file_name", expression_file_name}}; } return std::nullopt; } auto RepositoryConfig::RepositoryKey(Storage const& storage, std::string const& repo) const noexcept -> std::optional { auto const unique = DeduplicateRepo(repo, storage.GetHashFunction()); if (auto const* data = Data(unique)) { // compute key only once (thread-safe) return data->key.SetOnceAndGet( [this, &storage, &unique]() -> std::optional { if (auto graph = BuildGraphForRepository( unique, storage.GetHashFunction())) { auto const& cas = storage.CAS(); return cas.StoreBlob(graph->dump(2)); } return std::nullopt; }); } return std::nullopt; } // Obtain canonical name (according to bisimulation) for the given repository. auto RepositoryConfig::DeduplicateRepo(std::string const& repo, HashFunction hash_function) const -> std::string { // Compute duplicates map only once (thread-safe) auto const& duplicates = duplicates_.SetOnceAndGet([this, &hash_function] { // To detect duplicate repository descriptions, we represent each // repository as a DFA state with repo name as state name, repo // bindings as state transitions, and repo base description as state // content id. Then we use a DFA minimizer to find the bisimulation // for each state. auto minimizer = DFAMinimizer{}; for (auto const& [repo, data] : repos_) { // Only add content-fixed repositories. This is sufficient, as for // incomplete graphs our minimizer implementation identifies states // with transitions to differently-named missing states as // distinguishable. Even if those were considered indistinguishable, // repository key computation would still work correctly, as this // computation is only performed if all transitive dependencies are // content-fixed. if (data.base_desc) { // Use hash of content-fixed base description as content id auto hash = hash_function.PlainHashData(data.base_desc->dump()).Bytes(); // Add state with name, transitions, and content id minimizer.AddState(repo, data.info.name_mapping, hash); } } return minimizer.ComputeBisimulation(); }); // Lookup canonical name for given repository in duplicates map auto it = duplicates.find(repo); if (it != duplicates.end()) { return it->second; } return repo; } // Returns the repository-local representation of its dependency graph if all // contained repositories are content fixed or return std::nullopt otherwise. auto RepositoryConfig::BuildGraphForRepository(std::string const& repo, HashFunction hash_function) const -> std::optional { auto graph = nlohmann::json::object(); int id_counter{}; std::unordered_map repo_ids{}; if (AddToGraphAndGetId( &graph, &id_counter, &repo_ids, repo, hash_function)) { return graph; } return std::nullopt; } // Add given repository to the given graph and return its traversal specific // unique id if it and all its dependencies are content-fixed or return // std::nullopt otherwise. Recursion immediately aborts on traversing the first // non-content-fixed repository. auto RepositoryConfig::AddToGraphAndGetId( gsl::not_null const& graph, gsl::not_null const& id_counter, gsl::not_null*> const& repo_ids, std::string const& repo, HashFunction hash_function) const -> std::optional { auto const unique_repo = DeduplicateRepo(repo, hash_function); auto repo_it = repo_ids->find(unique_repo); if (repo_it != repo_ids->end()) { // The same or equivalent repository was already requested before return repo_it->second; } auto const* data = Data(unique_repo); if (data != nullptr and data->base_desc) { // Compute the unique id (traversal order) and store it auto repo_id = std::to_string((*id_counter)++); repo_ids->emplace(unique_repo, repo_id); // Compute repository description from content-fixed base // description and bindings to unique ids of depending repositories auto repo_desc = *data->base_desc; repo_desc["bindings"] = nlohmann::json::object(); for (auto const& [local_name, global_name] : data->info.name_mapping) { auto global_id = AddToGraphAndGetId( graph, id_counter, repo_ids, global_name, hash_function); if (not global_id) { return std::nullopt; } repo_desc["bindings"][local_name] = std::move(*global_id); } // Add repository description to graph with unique id as key (*graph)[repo_id] = std::move(repo_desc); return repo_id; } return std::nullopt; } void RepositoryConfig::SetPrecomputedRoot(PrecomputedRoot const& root, FileRoot const& value) { for (auto const& [name, desc] : repos_) { auto new_info = desc.info; bool changed = false; for (gsl::not_null candidate : {&new_info.workspace_root, &new_info.target_root, &new_info.rule_root, &new_info.expression_root}) { if (candidate->GetPrecomputedDescription() == root) { *candidate = value; changed = true; } } if (changed) { SetInfo(name, std::move(new_info)); } } } auto RepositoryConfig::ReadTreeFromGitCAS( std::string const& hex_id) const noexcept -> std::optional { if (git_cas_ == nullptr or storage_config_ == nullptr) { return std::nullopt; } return GitTreeUtils::ReadValidGitCASTree( *storage_config_, hex_id, git_cas_); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/repository_config.hpp000066400000000000000000000204141516554100600301510ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP #include #include #include #include #include #include #include #include // std::move #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/atomic_value.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" class RepositoryConfig { public: struct RepositoryInfo { // NOLINT(bugprone-exception-escape) FileRoot workspace_root; FileRoot target_root{workspace_root}; FileRoot rule_root{target_root}; FileRoot expression_root{rule_root}; std::map name_mapping{}; std::string target_file_name{"TARGETS"}; std::string rule_file_name{"RULES"}; std::string expression_file_name{"EXPRESSIONS"}; // Return base content description without bindings if all roots are // content fixed or return std::nullopt otherwise. [[nodiscard]] auto BaseContentDescription() const -> std::optional; }; void SetInfo(std::string const& repo, RepositoryInfo&& info) { repos_[repo].base_desc = info.BaseContentDescription(); repos_[repo].info = std::move(info); repos_[repo].key.Reset(); duplicates_.Reset(); } [[nodiscard]] auto SetGitCAS( std::filesystem::path const& repo_path, gsl::not_null const& storage_config, LogLevel log_level = LogLevel::Warning) noexcept -> bool { storage_config_ = storage_config; git_cas_ = GitCAS::Open(repo_path, log_level); return static_cast(git_cas_); } /// \brief Replace all entries of the precomputed root with an exact root. /// \param root Root to be replaced. /// \param value Root to be used as a replacement. void SetPrecomputedRoot(PrecomputedRoot const& root, FileRoot const& value); [[nodiscard]] auto Info(std::string const& repo) const noexcept -> RepositoryInfo const* { if (auto const* data = Data(repo)) { return &data->info; } return nullptr; } [[nodiscard]] auto ReadBlobFromGitCAS( std::string const& hex_id, bool is_symlink, LogLevel log_failure = LogLevel::Warning) const noexcept -> std::optional { return git_cas_ ? git_cas_->ReadObject(hex_id, /*is_hex_id=*/true, /*as_valid_symlink=*/is_symlink, log_failure) : std::nullopt; } [[nodiscard]] auto ReadTreeFromGitCAS( std::string const& hex_id) const noexcept -> std::optional; [[nodiscard]] auto WorkspaceRoot(std::string const& repo) const noexcept -> FileRoot const* { return Get( repo, [](auto const& info) { return &info.workspace_root; }); } [[nodiscard]] auto TargetRoot(std::string const& repo) const noexcept -> FileRoot const* { return Get( repo, [](auto const& info) { return &info.target_root; }); } [[nodiscard]] auto RuleRoot(std::string const& repo) const -> FileRoot const* { return Get(repo, [](auto const& info) { return &info.rule_root; }); } [[nodiscard]] auto ExpressionRoot(std::string const& repo) const noexcept -> FileRoot const* { return Get( repo, [](auto const& info) { return &info.expression_root; }); } [[nodiscard]] auto GlobalName(std::string const& repo, std::string const& local_name) const noexcept -> std::string const* { return Get( repo, [&local_name](auto const& info) -> std::string const* { auto it = info.name_mapping.find(local_name); if (it != info.name_mapping.end()) { return &it->second; } return nullptr; }); } [[nodiscard]] auto TargetFileName(std::string const& repo) const noexcept -> std::string const* { return Get( repo, [](auto const& info) { return &info.target_file_name; }); } [[nodiscard]] auto RuleFileName(std::string const& repo) const noexcept -> std::string const* { return Get( repo, [](auto const& info) { return &info.rule_file_name; }); } [[nodiscard]] auto ExpressionFileName( std::string const& repo) const noexcept -> std::string const* { return Get( repo, [](auto const& info) { return &info.expression_file_name; }); } // Obtain repository's cache key if the repository is content fixed or // std::nullopt otherwise. [[nodiscard]] auto RepositoryKey(Storage const& storage, std::string const& repo) const noexcept -> std::optional; // used for testing void Reset() { repos_.clear(); git_cas_.reset(); duplicates_.Reset(); } private: using duplicates_t = std::unordered_map; // All data we store per repository. struct RepositoryData { // Info structure (roots, file names, bindings) RepositoryInfo info{}; // Base description if content-fixed std::optional base_desc; // Cache key if content-fixed AtomicValue> key; }; std::unordered_map repos_; GitCASPtr git_cas_; StorageConfig const* storage_config_ = nullptr; AtomicValue duplicates_; template [[nodiscard]] auto Get(std::string const& repo, std::function const& getter) const noexcept -> T const* { try { if (auto const* info = Info(repo)) { return getter(*info); } } catch (...) { return nullptr; }; return nullptr; } [[nodiscard]] auto Data(std::string const& repo) const noexcept -> RepositoryData const* { auto it = repos_.find(repo); if (it != repos_.end()) { return &it->second; } return nullptr; } [[nodiscard]] auto DeduplicateRepo(std::string const& repo, HashFunction hash_function) const -> std::string; [[nodiscard]] auto BuildGraphForRepository(std::string const& repo, HashFunction hash_function) const -> std::optional; [[nodiscard]] auto AddToGraphAndGetId( gsl::not_null const& graph, gsl::not_null const& id_counter, gsl::not_null*> const& repo_ids, std::string const& repo, HashFunction hash_function) const -> std::optional; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/retry_cli.hpp000066400000000000000000000040001516554100600263720ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_CLI_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_CLI_HPP #include #include "CLI/CLI.hpp" #include "gsl/gsl" /// \brief Arguments required for tuning the retry strategy. struct RetryArguments { std::optional max_attempts; std::optional initial_backoff_seconds; std::optional max_backoff_seconds; }; static inline void SetupRetryArguments( gsl::not_null const& app, gsl::not_null const& args) { app->add_option( "--max-attempts", args->max_attempts, "Total number of attempts in case of a remote resource becomes " "unavailable. Must be greater than 0. (Default: 1)"); app->add_option( "--initial-backoff-seconds", args->initial_backoff_seconds, "Initial amount of time, in seconds, to wait before retrying a rpc " "call. The waiting time is doubled at each attempt. Must be greater " "than 0. (Default: 1)"); app->add_option("--max-backoff-seconds", args->max_backoff_seconds, "The backoff time cannot be bigger than this parameter. " "Note that some jitter is still added to avoid to overload " "the resources that survived the outage. (Default: 60)"); } #endif // INCLUDED_SRC_BUILDTOOL_COMMON_RETRY_CLI_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/statistics.hpp000066400000000000000000000104361516554100600266020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP #include class Statistics { public: void Reset() noexcept { num_actions_queued_ = 0; num_actions_executed_ = 0; num_actions_cached_ = 0; num_actions_flaky_ = 0; num_actions_flaky_tainted_ = 0; num_rebuilt_actions_compared_ = 0; num_rebuilt_actions_missing_ = 0; num_trees_analysed_ = 0; } void IncrementActionsQueuedCounter() noexcept { ++num_actions_queued_; } void IncrementActionsExecutedCounter() noexcept { ++num_actions_executed_; } void IncrementActionsCachedCounter() noexcept { ++num_actions_cached_; } void IncrementActionsFlakyCounter() noexcept { ++num_actions_flaky_; } void IncrementActionsFlakyTaintedCounter() noexcept { ++num_actions_flaky_tainted_; } void IncrementRebuiltActionMissingCounter() noexcept { ++num_rebuilt_actions_missing_; } void IncrementRebuiltActionComparedCounter() noexcept { ++num_rebuilt_actions_compared_; } void IncrementExportsCachedCounter() noexcept { ++num_exports_cached_; } void IncrementExportsUncachedCounter() noexcept { ++num_exports_uncached_; } void IncrementExportsNotEligibleCounter() noexcept { ++num_exports_not_eligible_; } void IncrementExportsFoundCounter() noexcept { ++num_exports_found_; } void IncrementExportsServedCounter() noexcept { ++num_exports_served_; } void IncrementTreesAnalysedCounter() noexcept { ++num_trees_analysed_; } [[nodiscard]] auto ActionsQueuedCounter() const noexcept -> int { return num_actions_queued_; } [[nodiscard]] auto ActionsExecutedCounter() const noexcept -> int { return num_actions_executed_; } [[nodiscard]] auto ActionsCachedCounter() const noexcept -> int { return num_actions_cached_; } [[nodiscard]] auto ActionsFlakyCounter() const noexcept -> int { return num_actions_flaky_; } [[nodiscard]] auto ActionsFlakyTaintedCounter() const noexcept -> int { return num_actions_flaky_tainted_; } [[nodiscard]] auto RebuiltActionMissingCounter() const noexcept -> int { return num_rebuilt_actions_missing_; } [[nodiscard]] auto RebuiltActionComparedCounter() const noexcept -> int { return num_rebuilt_actions_compared_; } [[nodiscard]] auto ExportsCachedCounter() const noexcept -> int { return num_exports_cached_; } [[nodiscard]] auto ExportsUncachedCounter() const noexcept -> int { return num_exports_uncached_; } [[nodiscard]] auto ExportsNotEligibleCounter() const noexcept -> int { return num_exports_not_eligible_; } [[nodiscard]] auto ExportsFoundCounter() const noexcept -> int { return num_exports_found_; } [[nodiscard]] auto ExportsServedCounter() const noexcept -> int { return num_exports_served_; } [[nodiscard]] auto TreesAnalysedCounter() const noexcept -> int { return num_trees_analysed_; } private: std::atomic num_actions_queued_; std::atomic num_actions_executed_; std::atomic num_actions_cached_; std::atomic num_actions_flaky_; std::atomic num_actions_flaky_tainted_; std::atomic num_rebuilt_actions_missing_; std::atomic num_rebuilt_actions_compared_; std::atomic num_exports_cached_; std::atomic num_exports_uncached_; std::atomic num_exports_not_eligible_; std::atomic num_exports_found_; std::atomic num_exports_served_; std::atomic num_trees_analysed_; }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/tree.hpp000066400000000000000000000070301516554100600253430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" // Describes tree, its inputs, output (tree artifact), and action (tree action). class Tree { using inputs_t = ActionDescription::inputs_t; public: using Ptr = std::shared_ptr; explicit Tree(inputs_t&& inputs) : id_{ComputeId(inputs)}, inputs_{std::move(inputs)} {} [[nodiscard]] auto Id() const& -> std::string const& { return id_; } [[nodiscard]] auto Id() && -> std::string { return std::move(id_); } [[nodiscard]] auto ToJson() const -> nlohmann::json { return ComputeDescription(inputs_); } [[nodiscard]] auto Inputs() const -> inputs_t { return inputs_; } [[nodiscard]] auto Action() const -> ActionDescription { return { {/*unused*/}, {/*unused*/}, Action::CreateTreeAction(id_), inputs_}; } [[nodiscard]] auto Output() const -> ArtifactDescription { return ArtifactDescription::CreateTree(id_); } [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, std::string const& id, nlohmann::json const& json) -> std::optional { auto inputs = inputs_t{}; inputs.reserve(json.size()); for (auto const& [path, artifact] : json.items()) { auto artifact_desc = ArtifactDescription::FromJson(hash_type, artifact); if (not artifact_desc) { return std::nullopt; } inputs.emplace(path, std::move(*artifact_desc)); } return std::make_shared(id, std::move(inputs)); } Tree(std::string id, inputs_t&& inputs) : id_{std::move(id)}, inputs_{std::move(inputs)} {} private: std::string id_; inputs_t inputs_; static auto ComputeDescription(inputs_t const& inputs) -> nlohmann::json { auto json = nlohmann::json::object(); for (auto const& [path, artifact] : inputs) { json[path] = artifact.ToJson(); } return json; } static auto ComputeId(inputs_t const& inputs) -> std::string { // The type of HashFunction is irrelevant here. It is used for // identification of trees. SHA256 is used. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; return hash_function .PlainHashData( fmt::format("TREE:{}", ComputeDescription(inputs).dump())) .HexString(); } }; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/tree_overlay.hpp000066400000000000000000000103421516554100600271040ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_TREE_OVERLAY_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_TREE_OVERLAY_HPP #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" class TreeOverlay { public: using to_overlay_t = std::vector; using Ptr = std::shared_ptr; explicit TreeOverlay(to_overlay_t const& trees, bool disjoint) : id_{ComputeId(trees, disjoint)}, trees_{trees}, disjoint_{disjoint} {} [[nodiscard]] auto Id() const& -> std::string const& { return id_; } [[nodiscard]] auto Id() && -> std::string { return std::move(id_); } [[nodiscard]] auto ToJson() const -> nlohmann::json { return ComputeDescription(trees_, disjoint_); } [[nodiscard]] auto Inputs() const -> ActionDescription::inputs_t { return AsInputs(trees_); } [[nodiscard]] auto Action() const -> ActionDescription { return {{/*unused*/}, {/*unused*/}, Action::CreateTreeOverlayAction(id_, disjoint_), AsInputs(trees_)}; } [[nodiscard]] auto Output() const -> ArtifactDescription { return ArtifactDescription::CreateTreeOverlay(id_); } [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, std::string const& id, nlohmann::json const& json) -> std::optional { bool disjoint = json["disjoint"]; auto trees = to_overlay_t{}; for (auto const& entry : json["trees"]) { auto artifact_desc = ArtifactDescription::FromJson(hash_type, entry); if (not artifact_desc) { return std::nullopt; } trees.emplace_back(std::move(*artifact_desc)); } return std::make_shared(id, std::move(trees), disjoint); } TreeOverlay(std::string id, to_overlay_t&& trees, bool disjoint) : id_{std::move(id)}, trees_{std::move(trees)}, disjoint_{disjoint} {} private: std::string id_; to_overlay_t trees_; bool disjoint_; static auto ComputeDescription(to_overlay_t const& trees, bool disjoint) -> nlohmann::json { auto json = nlohmann::json::object(); auto tree_descs = nlohmann::json::array(); for (auto const& tree : trees) { tree_descs.emplace_back(tree.ToJson()); } json["trees"] = tree_descs; json["disjoint"] = disjoint; return json; } static auto ComputeId(to_overlay_t const& trees, bool disjoint) -> std::string { // The type of HashFunction is irrelevant here. It is used for // identification of trees. SHA256 is used. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; return hash_function .PlainHashData(fmt::format( "TREE_OVERLAY:{}", ComputeDescription(trees, disjoint).dump())) .HexString(); } static auto AsInputs(to_overlay_t const& trees) -> ActionDescription::inputs_t { ActionDescription::inputs_t as_inputs{}; for (size_t i = 0; i < trees.size(); i++) { as_inputs.emplace(fmt::format("{:010d}", i), trees[i]); } return as_inputs; } }; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/common/user_structs.hpp000066400000000000000000000052551516554100600271600ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP #define INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/main/constants.hpp" /* Structures populated exclusively from command line with user-defined data */ static inline const std::filesystem::path kDefaultSetupRoot = FileSystemManager::GetCurrentDirectory(); struct LocalPaths { // user-defined locations std::optional root{std::nullopt}; std::filesystem::path setup_root = kDefaultSetupRoot; std::optional workspace_root{ // find workspace root []() -> std::optional { std::function is_workspace_root = [&](std::filesystem::path const& path) { return std::any_of( kRootMarkers.begin(), kRootMarkers.end(), [&path](auto const& marker) { return FileSystemManager::Exists(path / marker); }); }; // default path to current dir auto path = kDefaultSetupRoot; auto root_path = path.root_path(); while (true) { if (is_workspace_root(path)) { return path; } if (path == root_path) { return std::nullopt; } path = path.parent_path(); } }()}; nlohmann::json git_checkout_locations; std::vector distdirs; }; struct CAInfo { bool no_ssl_verify{false}; std::optional ca_bundle{std::nullopt}; }; using LocalPathsPtr = std::shared_ptr; using CAInfoPtr = std::shared_ptr; #endif // INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/000077500000000000000000000000001516554100600254515ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/TARGETS000066400000000000000000000135571516554100600265200ustar00rootroot00000000000000{ "analyse_and_build": { "type": ["@", "rules", "CC", "library"] , "name": ["analyse_and_build"] , "hdrs": ["analyse_and_build.hpp"] , "srcs": ["analyse_and_build.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "analyse"] , ["src/buildtool/main", "analyse_context"] ] , "stage": ["src", "buildtool", "computed_roots"] , "private-deps": [ ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/main", "build_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "storage"] ] } , "inquire_serve": { "type": ["@", "rules", "CC", "library"] , "name": ["inquire_serve"] , "hdrs": ["inquire_serve.hpp"] , "srcs": ["inquire_serve.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "analyse_context"] ] , "private-deps": [ "artifacts_root" , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/base_maps", "module_name"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "computed_roots"] } , "artifacts_root": { "type": ["@", "rules", "CC", "library"] , "name": ["artifacts_root"] , "hdrs": ["artifacts_root.hpp"] , "srcs": ["artifacts_root.cpp"] , "stage": ["src", "buildtool", "computed_roots"] , "deps": [ ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/multithreading", "async_map_consumer"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/common", "common"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] ] } , "roots_progress": { "type": ["@", "rules", "CC", "library"] , "name": ["roots_progress"] , "hdrs": ["roots_progress.hpp"] , "srcs": ["roots_progress.cpp"] , "stage": ["src", "buildtool", "computed_roots"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/progress_reporting", "task_tracker"] ] , "private-deps": [["@", "fmt", "", "fmt"], ["src/buildtool/logging", "log_level"]] } , "evaluate": { "type": ["@", "rules", "CC", "library"] , "name": ["evaluate"] , "hdrs": ["evaluate.hpp"] , "private-hdrs": ["lookup_cache.hpp"] , "srcs": ["evaluate.cpp", "lookup_cache.cpp"] , "stage": ["src", "buildtool", "computed_roots"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_engine/executor", "context"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] ] , "private-deps": [ "analyse_and_build" , "artifacts_root" , "inquire_serve" , "roots_progress" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/base_maps", "field_reader"] , ["src/buildtool/build_engine/base_maps", "module_name"] , ["src/buildtool/build_engine/base_maps", "targets_file_map"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/file_system", "precomputed_root"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "analyse_context"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/tree_structure", "tree_structure_utils"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "tmp_dir"] , ["src/utils/cpp", "vector"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/analyse_and_build.cpp000066400000000000000000000074441516554100600316230ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/analyse_and_build.hpp" #include #include #include #include #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/storage.hpp" [[nodiscard]] auto AnalyseAndBuild( gsl::not_null const& analyse_context, GraphTraverser const& traverser, BuildMaps::Target::ConfiguredTarget const& id, std::size_t jobs, gsl::not_null const& apis, Logger const* logger) -> std::optional { auto analysis_result = AnalyseTarget(analyse_context, id, jobs, /*request_action_input*/ std::nullopt, logger); if (not analysis_result) { if (logger != nullptr) { logger->Emit(LogLevel::Warning, "Failed to analyse target {}", id.ToString()); } return std::nullopt; } if (logger != nullptr) { logger->Emit(LogLevel::Info, "Analysed target {}", id.ToString()); } auto const [artifacts, runfiles] = ReadOutputArtifacts(analysis_result->target); auto [actions, blobs, trees, tree_overlays] = analysis_result->result_map.ToResult( analyse_context->statistics, analyse_context->progress, logger); auto const cache_targets = analysis_result->result_map.CacheTargets(); // Clean up analyse_result map, now that it is no longer needed { TaskSystem ts{jobs}; analysis_result->result_map.Clear(&ts); } auto extra_artifacts = CollectNonKnownArtifacts(cache_targets); for (auto const& [name, desc] : runfiles) { extra_artifacts.emplace_back(desc); } auto build_result = traverser.BuildAndStage(artifacts, {}, std::move(actions), std::move(blobs), std::move(trees), std::move(tree_overlays), std::move(extra_artifacts)); if (not build_result) { if (logger != nullptr) { logger->Emit( LogLevel::Warning, "Build for target {} failed", id.ToString()); } return std::nullopt; } WriteTargetCacheEntries(cache_targets, build_result->extra_infos, jobs, *apis, TargetCacheWriteStrategy::Sync, analyse_context->storage->TargetCache(), logger); return AnalyseAndBuildResult{.analysis_result = *std::move(analysis_result), .build_result = *std::move(build_result)}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/analyse_and_build.hpp000066400000000000000000000033441516554100600316230ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_ANALYSE_AND_BUILD_HPP #define INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_ANALYSE_AND_BUILD_HPP #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/analyse.hpp" #include "src/buildtool/main/analyse_context.hpp" struct AnalyseAndBuildResult { AnalysisResult analysis_result; GraphTraverser::BuildResult build_result; }; // Analyse and build a given target; however, install only the artifacts, // ignoring the runfiles. [[nodiscard]] auto AnalyseAndBuild( gsl::not_null const& analyse_context, GraphTraverser const& traverser, BuildMaps::Target::ConfiguredTarget const& id, std::size_t jobs, gsl::not_null const& apis, Logger const* logger = nullptr) -> std::optional; #endif // INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_ANALYSE_AND_BUILD_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/artifacts_root.cpp000066400000000000000000000153221516554100600312030ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/artifacts_root.hpp" #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { void AddEntryRaw(gsl::not_null const& tree, std::string const& p, std::string const& raw_hash, ObjectType const& ot) { if (tree->contains(raw_hash)) { (*tree)[raw_hash].emplace_back(p, ot); } else { (*tree)[raw_hash] = std::vector{GitRepo::TreeEntry{p, ot}}; } } auto AddEntry(gsl::not_null const& tree, std::string const& p, std::string const& hash, ObjectType const& ot, AsyncMapConsumerLoggerPtr const& logger) -> bool { auto raw_hash = FromHexString(hash); if (not raw_hash) { (*logger)( fmt::format("Not a hex string {}", nlohmann::json(hash).dump()), true); return false; } AddEntryRaw(tree, p, *raw_hash, ot); return true; } // Structure building a git tree from entries traversed in order so that // all entries of a subdirectory come next to each other. struct PartialTree { std::filesystem::path current_dir_; std::stack partial_trees_; PartialTree() { partial_trees_.emplace(); } void Down(std::string const& segment) { if (segment == "." or segment.empty()) { return; } partial_trees_.emplace(); current_dir_ = current_dir_ / segment; } auto Up() -> bool { auto name = current_dir_.filename().string(); auto git_tree = GitRepo::CreateShallowTree(partial_trees_.top()); if (not git_tree) { return false; } current_dir_ = current_dir_.parent_path(); partial_trees_.pop(); AddEntryRaw( &partial_trees_.top(), name, git_tree->first, ObjectType::Tree); return true; } auto To(std::filesystem::path const& dir) -> bool { auto rel_path = dir.lexically_relative(current_dir_); while (*rel_path.begin() == "..") { if (not Up()) { return false; } rel_path = dir.lexically_relative(current_dir_); } for (auto const& segment : rel_path) { Down(segment.string()); } return true; } auto Add(std::string const& ps, std::string const& hash, ObjectType const& ot, AsyncMapConsumerLoggerPtr const& logger) -> bool { auto p = std::filesystem::path(ps); if (not To(p.parent_path())) { (*logger)("Failure in tree computation", true); return false; } return AddEntry( &partial_trees_.top(), p.filename().string(), hash, ot, logger); } auto Done(AsyncMapConsumerLoggerPtr const& logger) -> std::optional { while (not current_dir_.empty()) { if (not Up()) { (*logger)("Failure computing final git tree", true); return std::nullopt; } } auto git_tree = GitRepo::CreateShallowTree(partial_trees_.top()); if (not git_tree) { (*logger)("Failure computing top-level git tree", true); return std::nullopt; } return ToHexString(git_tree->first); } }; } // namespace auto ArtifactsRoot(ExpressionPtr const& stage, AsyncMapConsumerLoggerPtr const& logger, std::optional const& rehash) -> std::optional { if (not stage->IsMap()) { (*logger)(fmt::format("Expected stage to be a map, but found {}", stage->ToString()), true); return std::nullopt; } // We obtain the entries odered by key; in particular, the entries of all // subtrees will be next to each other. So, we compute the final tree // keeping a stack of partially set up tree while walking. auto partial_tree = PartialTree(); for (auto const& [ps, entry] : stage->Map().Items()) { if (not entry->IsArtifact()) { (*logger)(fmt::format("Expected stage, but at entry {} found {}", nlohmann::json(ps).dump(), entry->ToString()), true); return std::nullopt; } auto const& val = entry->Artifact(); if (not val.IsKnown()) { (*logger)(fmt::format( "Expected evaluated stage, but at entry {} found {}", nlohmann::json(ps).dump(), entry->ToString()), true); return std::nullopt; } auto info_opt = val.ToArtifact().Info(); if (not info_opt) { (*logger)(fmt::format("Failed to determine artifact info of known " "artifact {}", entry->ToString()), true); return std::nullopt; } auto info = *info_opt; if (rehash) { auto rehash_result = rehash->Rehash(info); if (not rehash_result) { (*logger)( fmt::format("Rehashing failed: {}", rehash_result.error()), true); return std::nullopt; } info = *rehash_result; } if (not partial_tree.Add(ps, info.digest.hash(), info.type, logger)) { return std::nullopt; } } return partial_tree.Done(logger); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/artifacts_root.hpp000066400000000000000000000030651516554100600312110ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_ARTIFACTS_ROOT_HPP #define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_ARTIFACTS_ROOT_HPP #include #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" /// \brief Compute to the git tree identifier, as hex string, corresponding to /// an artifact stage; return std::nullopt in case of errors /// \param stage Expression pointer supposed to represent a map from logical /// paths to known artifacts /// \param logger Logger to report problems; will be called with the fatal /// property in case of error auto ArtifactsRoot(ExpressionPtr const& stage, AsyncMapConsumerLoggerPtr const& logger, std::optional const& rehash = std::nullopt) -> std::optional; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/evaluate.cpp000066400000000000000000001053611516554100600277710ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/evaluate.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/computed_roots/analyse_and_build.hpp" #include "src/buildtool/computed_roots/inquire_serve.hpp" #include "src/buildtool/computed_roots/lookup_cache.hpp" #include "src/buildtool/computed_roots/roots_progress.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" #include "src/buildtool/logging/log_sink_file.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/tree_structure/tree_structure_utils.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "src/utils/cpp/vector.hpp" namespace { // Add the description of a computed root to a vector, if the given // root is computed. void AddDescriptionIfPrecomputed( FileRoot const& root, gsl::not_null*> croots) { if (auto desc = root.GetPrecomputedDescription()) { croots->emplace_back(*std::move(desc)); } } // Traverse, starting from a given repository, in order to find the // computed roots it depends on. void TraverseRepoForComputedRoots( std::string const& name, gsl::not_null const& repository_config, gsl::not_null*> roots, gsl::not_null*> seen) { if (seen->contains(name)) { return; } seen->insert(name); const auto* info = repository_config->Info(name); if (info == nullptr) { Logger::Log(LogLevel::Warning, "Ignoring unknown repository {} while determining the " "derived roots to compute", nlohmann::json(name).dump()); return; } AddDescriptionIfPrecomputed(info->workspace_root, roots); AddDescriptionIfPrecomputed(info->target_root, roots); AddDescriptionIfPrecomputed(info->rule_root, roots); AddDescriptionIfPrecomputed(info->expression_root, roots); for (auto const& [k, v] : info->name_mapping) { TraverseRepoForComputedRoots(v, repository_config, roots, seen); } } // For a given repository, return the list of computed roots it directly depends // upon auto GetRootDeps(std::string const& name, gsl::not_null const& repository_config) -> std::vector { std::vector result{}; std::set seen{}; TraverseRepoForComputedRoots(name, repository_config, &result, &seen); sort_and_deduplicate(&result); Logger::Log(LogLevel::Debug, [&]() { std::ostringstream msg{}; msg << "Roots for " << nlohmann::json(name) << ", total of " << result.size() << ":"; for (auto const& root : result) { msg << "\n - " << root.ToString(); } return msg.str(); }); return result; } // For each computed root, we have to determine the git tree identifier; it has // to be given as string, as this is the format needed to update a repository // root. using RootMap = AsyncMapConsumer; auto WhileHandling(PrecomputedRoot const& root, AsyncMapConsumerLoggerPtr const& logger) -> AsyncMapConsumerLoggerPtr { return std::make_shared( [root, logger](auto const& msg, auto fatal) { (*logger)(fmt::format( "While materializing {}:\n{}", root.ToString(), msg), fatal); }); } void ComputeAndFill( ComputedRoot const& key, gsl::not_null const& repository_config, gsl::not_null const& traverser_args, ServeApi const* serve, gsl::not_null const& context, gsl::not_null const& storage_config, gsl::not_null*> const& rehash, gsl::not_null const& config_lock, gsl::not_null const& git_lock, gsl::not_null const& root_stats, gsl::not_null const& root_tasks, std::size_t jobs, RootMap::LoggerPtr const& logger, RootMap::SetterPtr const& setter) { auto tmpdir = storage_config->CreateTypedTmpDir("computed-root"); if (not tmpdir) { (*logger)("Failed to create temporary directory", true); return; } auto root_dir = tmpdir->GetPath() / "root"; auto log_file = tmpdir->GetPath() / "log"; auto storage = Storage::Create(storage_config); auto statistics = Statistics(); auto progress = Progress(); auto reporter = BaseProgressReporter::Reporter([]() {}); BuildMaps::Target::ConfiguredTarget target{ BuildMaps::Base::EntityName{ key.repository, key.target_module, key.target_name}, Configuration{Expression::FromJson(key.config)}}; AnalyseContext analyse_context{.repo_config = repository_config, .storage = &storage, .statistics = &statistics, .progress = &progress, .serve = serve}; root_tasks->Start(target.ToString()); root_stats->IncrementActionsQueuedCounter(); Logger build_logger = Logger( target.ToString(), std::vector{LogSinkFile::CreateFactory(log_file)}); auto root_build_args = *traverser_args; root_build_args.stage = StageArguments{.output_dir = root_dir, .remember = true}; root_build_args.rebuild = std::nullopt; root_build_args.build.print_to_stdout = std::nullopt; root_build_args.build.print_unique = false; root_build_args.build.dump_artifacts = std::vector{}; root_build_args.build.show_runfiles = false; auto root_exec_context = ExecutionContext{context->repo_config, context->apis, context->remote_context, &statistics, &progress, std::nullopt}; auto cache_lookup = expected, std::monostate>(std::nullopt); { std::shared_lock computing{*config_lock}; cache_lookup = LookupCache(target, repository_config, storage, logger, *rehash); } if (not cache_lookup) { // prerequisite failure; fatal logger call already handled by // LookupCache return; } if (*cache_lookup) { root_stats->IncrementActionsCachedCounter(); root_tasks->Stop(target.ToString()); std::string root = **cache_lookup; auto tree_present = GitRepo::IsTreeInRepo(storage_config->GitRoot(), root); if (not tree_present) { (*logger)(fmt::format("While checking presence of tree {} in local " "git repo:\n{}", root, std::move(tree_present).error()), /*fatal=*/true); return; } if (*tree_present) { Logger::Log(LogLevel::Performance, "Root {} taken from cache to be {}", target.ToString(), root); auto root_result = FileRoot::FromGit( storage_config, storage_config->GitRoot(), root); if (not root_result) { (*logger)(fmt::format("Failed to create git root for {}", root), /*fatal=*/true); return; } { std::unique_lock setting{*config_lock}; repository_config->SetPrecomputedRoot(PrecomputedRoot{key}, *root_result); } (*setter)(std::move(root)); return; } } if (key.absent) { if (storage_config->hash_function.GetType() != HashFunction::Type::GitSHA1) { Logger::Log(LogLevel::Performance, "Computing root {} locally as rehashing would have to " "be done locally", key.ToString()); } else { auto serve_result = InquireServe(&analyse_context, target, &build_logger); if (serve_result) { auto root_result = FileRoot(*serve_result); Logger::Log(LogLevel::Performance, "Absent root {} obtained from serve to be {}", target.ToString(), *serve_result); { std::unique_lock setting{*config_lock}; repository_config->SetPrecomputedRoot(PrecomputedRoot{key}, root_result); } root_stats->IncrementExportsServedCounter(); root_tasks->Stop(target.ToString()); (*setter)(std::move(*serve_result)); return; } } } GraphTraverser traverser{ root_build_args, &root_exec_context, reporter, &build_logger}; std::optional build_result{}; { std::shared_lock computing{*config_lock}; build_result = AnalyseAndBuild(&analyse_context, traverser, target, jobs, context->apis, &build_logger); } auto log_blob = storage.CAS().StoreBlob(log_file, false); std::string log_desc{}; if (not log_blob) { (*logger)(fmt::format("Failed to store log file {} to CAS", log_file.string()), false); log_desc = "???"; } else { log_desc = log_blob->hash(); if (not build_result) { // synchronize the build log to a remote endpoint, if specified if (not context->apis->local->RetrieveToCas( {Artifact::ObjectInfo{.digest = *log_blob, .type = ObjectType::File, .failed = false}}, *context->apis->remote)) { (*logger)(fmt::format("Failed to upload build log {} to remote", log_desc), false); } } } if (not build_result) { (*logger)(fmt::format("Build failed, see {} for details", log_desc), true); return; } auto result = GitRepo::ImportToGit( *storage_config, root_dir, "computed root", git_lock); if (not result) { (*logger)(std::move(result).error(), /*fatal=*/true); return; } root_stats->IncrementActionsExecutedCounter(); root_tasks->Stop(target.ToString()); Logger::Log(LogLevel::Performance, "Root {} evaluated to {}, log {}", target.ToString(), *result, log_desc); auto root_result = FileRoot::FromGit(storage_config, storage_config->GitRoot(), *result); if (not root_result) { (*logger)(fmt::format("Failed to create git root for {}", *result), /*fatal=*/true); return; } if (key.absent) { if (serve == nullptr) { (*logger)(fmt::format("Requested root {} to be absent, without " "providing serve endpoint", key.ToString()), /*fatal=*/true); return; } auto known = serve->CheckRootTree(*result); if (known.has_value() and not *known) { auto tree_digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, *result, /*size_unknown=*/0, /*is_tree=*/true); if (not tree_digest) { (*logger)( fmt::format("Internal error getting digest for tree {}: {}", *known, tree_digest.error()), /*fatal=*/true); return; } auto uploaded_to_serve = serve->UploadTree(*tree_digest, storage_config->GitRoot()); if (not uploaded_to_serve) { (*logger)(fmt::format("Failed to sync {} to serve:{}", *result, uploaded_to_serve.error().Message()), /*fatal=*/true); return; } Logger::Log(LogLevel::Performance, "Uploaded {} to serve", *result); known = true; } if (not known.has_value() or not *known) { (*logger)( fmt::format("Failed to ensure {} is known to serve", *result), /*fatal=*/true); return; } } { // For setting, we need an exclusiver lock; so get one after we // dropped the shared one std::unique_lock setting{*config_lock}; repository_config->SetPrecomputedRoot(PrecomputedRoot{key}, *root_result); } (*setter)(*std::move(result)); } /// \brief Compute tree structure of the given root and return the resolved real /// root. /// There are a number of scenarios: /// 1. (LOCAL-LOCAL) Local tree structure of a local root: /// Finds the source tree locally and computes tree structure. After /// this evaluation the tree structure is present in local native CAS, in /// git repository, and in the TreeStructureCache. /// 2. (LOCAL-ABSENT) Local tree structure of an absent root: /// 2.1 First performs LOCAL-LOCAL using root's git identifier. This might /// minimize the network traffic if the source tree is present locally /// somewhere. /// 2.2 If fails, asks serve to compute the tree structure and performs /// LOCAL-LOCAL using tree_structure's git identifier. This might minimize /// the traffic as well since the tree structure may be already present /// locally. /// 2.3 If fails, downloads the tree from serve via the remote end point /// (with a possible rehashing), and runs LOCAL-LOCAL on this tree_structure /// to make the tree structure available for roots. /// 3. (ABSENT-ABSENT) Absent tree of an absent root: /// Compute absent tree structure on serve. /// 4. (ABSENT-LOCAL): Absent tree structure of a local root: /// Perform logic of LOCAL-LOCAL (compute tree structure locally) and /// upload the result of computation to serve. [[nodiscard]] auto ResolveTreeStructureRoot( TreeStructureRoot const& key, gsl::not_null const& repository_config, ServeApi const* serve, gsl::not_null const& storage_config, gsl::not_null const& config_lock, gsl::not_null const& git_lock) -> expected { // Obtain the file root that the key root is referring to FileRoot ref_root; { std::shared_lock lock{*config_lock}; auto const* root = repository_config->WorkspaceRoot(key.repository); if (root == nullptr) { return unexpected{fmt::format( "Failed to get referenced repository for {}", key.ToString())}; } ref_root = *root; } auto const hash = ref_root.GetTreeHash(); if (not hash) { return unexpected{ fmt::format("Failed to get the hash of the referenced git " "tree for {}", key.ToString())}; } auto const digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, *hash, /*size_unknown=*/0, /*is_tree=*/true); if (not digest) { return unexpected{digest.error()}; } // Create a native storage config if needed // Since tree structure works with git trees, a native storage is required. std::optional substitution_storage_config; if (not ProtocolTraits::IsNative(storage_config->hash_function.GetType())) { auto config = StorageConfig::Builder::Rebuild(*storage_config) .SetHashType(HashFunction::Type::GitSHA1) .Build(); if (not config) { return unexpected{ fmt::format("Failed to create a native storage config for {}", key.ToString())}; } substitution_storage_config.emplace(*config); } StorageConfig const& native_storage_config = substitution_storage_config.has_value() ? substitution_storage_config.value() : *storage_config; std::vector known_repositories{native_storage_config.GitRoot()}; if (not ref_root.IsAbsent()) { auto const path_to_git_cas = ref_root.GetGitCasRoot(); if (not path_to_git_cas) { return unexpected{ fmt::format("Failed to get the path to the git cas for {}", key.ToString())}; } known_repositories.push_back(*path_to_git_cas); } bool const compute_locally = not key.absent or not ref_root.IsAbsent(); std::optional local_tree_structure; // Try to compute the tree structure locally: if (compute_locally) { auto from_local = TreeStructureUtils::ComputeStructureLocally( *digest, known_repositories, native_storage_config, git_lock); if (not from_local.has_value()) { // A critical error occurred: return unexpected{std::move(from_local).error()}; } local_tree_structure = *from_local; } // For absent roots ask serve to process the tree: std::optional absent_tree_structure; if (not local_tree_structure.has_value() and ref_root.IsAbsent()) { if (serve == nullptr) { return unexpected{fmt::format( "No serve end point is given to compute {}", key.ToString())}; } auto on_serve = serve->ComputeTreeStructure(*digest); if (not on_serve.has_value()) { return unexpected{ fmt::format("Failed to compute {} on serve", key.ToString())}; } absent_tree_structure = *on_serve; } // Try to process an absent tree structure locally. It might be found in // CAS or git cache, so there'll be no need to download it from the // remote end point: if (compute_locally and not local_tree_structure.has_value() and absent_tree_structure.has_value()) { if (auto from_local = TreeStructureUtils::ComputeStructureLocally( *absent_tree_structure, known_repositories, native_storage_config, git_lock)) { local_tree_structure = *from_local; } else { // A critical error occurred: return unexpected{std::move(from_local).error()}; } // Failed to process absent tree structure locally, download artifacts // from remote: if (not local_tree_structure.has_value()) { auto downloaded = serve->DownloadTree(*absent_tree_structure); if (not downloaded.has_value()) { return unexpected{std::move(downloaded).error()}; } Logger::Log(LogLevel::Performance, "Root {} has been downloaded from the remote end point", key.ToString()); // The tree structure has been downloaded successfully, try to // resolve the root one more time: if (auto from_local = TreeStructureUtils::ComputeStructureLocally( *absent_tree_structure, known_repositories, native_storage_config, git_lock)) { local_tree_structure = *from_local; } else { // A critical error occurred: return unexpected{std::move(from_local).error()}; } } else { Logger::Log(LogLevel::Performance, "Root {} has been taken from local cache", key.ToString()); } } if (key.absent and absent_tree_structure.has_value()) { Logger::Log(LogLevel::Performance, "Root {} was computed on serve", key.ToString()); return FileRoot(absent_tree_structure->hash()); } if (key.absent and local_tree_structure.has_value()) { if (serve == nullptr) { return unexpected{fmt::format( "No serve end point is given to compute {}", key.ToString())}; } // Make sure the tree structure is available on serve: auto known = serve->CheckRootTree(local_tree_structure->hash()); if (known.has_value() and not *known) { auto uploaded = serve->UploadTree(*local_tree_structure, native_storage_config.GitRoot()); if (not uploaded.has_value()) { return unexpected{std::move(uploaded).error().Message()}; } known = true; } if (not known.has_value() or not *known) { return unexpected{ fmt::format("Failed to ensure that tree {} is " "available on serve", local_tree_structure->hash())}; } Logger::Log(LogLevel::Performance, "Root {} was computed locally and uploaded to serve", key.ToString()); return FileRoot(local_tree_structure->hash()); } if (local_tree_structure.has_value()) { auto resolved_root = FileRoot::FromGit(storage_config, native_storage_config.GitRoot(), local_tree_structure->hash()); if (not resolved_root) { return unexpected{ fmt::format("Failed to create root for {}", key.ToString())}; } return *resolved_root; } return unexpected{fmt::format("Failed to calculate tree structure for {}", key.ToString())}; } void ComputeTreeStructureAndFill( TreeStructureRoot const& key, gsl::not_null const& repository_config, ServeApi const* serve, gsl::not_null const& storage_config, gsl::not_null const& config_lock, gsl::not_null const& git_lock, RootMap::LoggerPtr const& logger, RootMap::SetterPtr const& setter) { auto resolved_root = ResolveTreeStructureRoot( key, repository_config, serve, storage_config, config_lock, git_lock); if (not resolved_root) { std::invoke(*logger, std::move(resolved_root).error(), /*fatal=*/true); return; } { // For setting, we need an exclusive lock; so get one after we // dropped the shared one std::unique_lock setting{*config_lock}; repository_config->SetPrecomputedRoot(PrecomputedRoot{key}, *resolved_root); } (*setter)(*std::move(resolved_root)->GetTreeHash()); } auto FillRoots( std::size_t jobs, gsl::not_null const& repository_config, gsl::not_null const& traverser_args, gsl::not_null const& context, ServeApi const* serve, gsl::not_null const& storage_config, gsl::not_null*> const& rehash, gsl::not_null const& config_lock, gsl::not_null const& git_lock, gsl::not_null const& stats, gsl::not_null const& tasks) -> RootMap { RootMap::ValueCreator fill_roots = [storage_config, rehash, repository_config, traverser_args, serve, context, config_lock, git_lock, stats, tasks, jobs](auto /*ts*/, auto setter, auto logger, auto subcaller, PrecomputedRoot const& key) { auto annotated_logger = WhileHandling(key, logger); std::vector roots{}; { std::shared_lock reading{*config_lock}; roots = GetRootDeps(key.GetReferencedRepository(), repository_config); } (*subcaller)( roots, [key, repository_config, traverser_args, context, serve, storage_config, config_lock, rehash, git_lock, jobs, stats, tasks, logger = annotated_logger, setter](auto /*values*/) { if (auto const computed = key.AsComputed()) { ComputeAndFill(*computed, repository_config, traverser_args, serve, context, storage_config, rehash, config_lock, git_lock, stats, tasks, jobs, logger, setter); } if (auto const tree_structure = key.AsTreeStructure()) { ComputeTreeStructureAndFill(*tree_structure, repository_config, serve, storage_config, config_lock, git_lock, logger, setter); } }, annotated_logger); }; return RootMap{fill_roots, jobs}; } } // namespace auto EvaluatePrecomputedRoots( gsl::not_null const& repository_config, std::string const& main_repo, ServeApi const* serve, StorageConfig const& storage_config, GraphTraverser::CommandLineArguments const& traverser_args, gsl::not_null const& context, std::size_t jobs) -> bool { auto roots = GetRootDeps(main_repo, repository_config); if (not roots.empty()) { Logger::Log(LogLevel::Info, "Repository {} depends on {} top-level computed roots", nlohmann::json(main_repo).dump(), roots.size()); // First, ensure the local git repository is present if (not FileSystemManager::CreateDirectory(storage_config.GitRoot())) { Logger::Log(LogLevel::Error, "Failed to create directory {}", storage_config.GitRoot().string()); return false; } if (not GitRepo::InitAndOpen(storage_config.GitRoot(), /*is_bare=*/true)) { Logger::Log(LogLevel::Error, "Failed to init and open git repository {}", storage_config.GitRoot().string()); return false; } // Prepare rehash-function, if rehashing is required std::optional rehash; if (not ProtocolTraits::IsNative( storage_config.hash_function.GetType())) { auto native_storage_config = StorageConfig::Builder::Rebuild(storage_config) .SetHashType(HashFunction::Type::GitSHA1) .Build(); if (not native_storage_config) { Logger::Log( LogLevel::Error, "Failed to create native storage config for rehashing:\n{}", std::move(native_storage_config).error()); return false; } rehash.emplace(storage_config, *std::move(native_storage_config), context->apis); } // Our RepositoryConfig is a bit problematic: it cannot be copied, hence // we have to change in place. Moreover, it is tread-safe for read // access, but not for writing, so we have to synchronize access out of // bound. std::shared_mutex repo_config_access{}; std::mutex git_access{}; Statistics stats{}; TaskTracker root_tasks{}; auto root_map = FillRoots(jobs, repository_config, &traverser_args, context, serve, &storage_config, &rehash, &repo_config_access, &git_access, &stats, &root_tasks); std::atomic done = false; std::atomic failed = false; std::atomic build_done = false; std::condition_variable cv{}; auto reporter = RootsProgress::Reporter(&stats, &root_tasks); auto observer = std::thread( [&reporter, &build_done, &cv] { reporter(&build_done, &cv); }); { TaskSystem ts{jobs}; root_map.ConsumeAfterKeysReady( &ts, roots, [&roots, &done](auto values) { Logger::Log(LogLevel::Debug, [&]() { std::ostringstream msg{}; msg << "Root building completed; top-level computed " "roots"; for (int i = 0; i < roots.size(); i++) { auto const& root = roots[i]; msg << "\n - " << root.ToString() << " evaluates to " << *values[i]; } return msg.str(); }); done = true; }, [&failed](auto const& msg, bool fatal) { Logger::Log( fatal ? LogLevel::Error : LogLevel::Warning, "While materializing top-level computed roots:\n{}", msg); failed = failed or fatal; } ); } build_done = true; cv.notify_all(); observer.join(); if (failed) { return false; } if (not done) { const std::function k_root_printer = [](PrecomputedRoot const& x) -> std::string { return x.ToString(); }; auto cycle_msg = DetectAndReportCycle( "computed roots", root_map, k_root_printer); if (cycle_msg) { Logger::Log(LogLevel::Error, "{}", *cycle_msg); } else { DetectAndReportPending( "computed roots", root_map, k_root_printer); } return false; } return true; } return true; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/evaluate.hpp000066400000000000000000000026251516554100600277750ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_EVALUTE_HPP #define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_EVALUTE_HPP #include #include #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" auto EvaluatePrecomputedRoots( gsl::not_null const& repository_config, std::string const& main_repo, ServeApi const* serve, StorageConfig const& storage_config, GraphTraverser::CommandLineArguments const& traverser_args, gsl::not_null const& context, std::size_t jobs) -> bool; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/inquire_serve.cpp000066400000000000000000000127441516554100600310450ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/inquire_serve.hpp" #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/computed_roots/artifacts_root.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" [[nodiscard]] auto InquireServe( gsl::not_null const& analyse_context, BuildMaps::Target::ConfiguredTarget const& id, gsl::not_null const& logger) -> std::optional { auto const& repo_name = id.target.ToModule().repository; if (not analyse_context->repo_config->TargetRoot(repo_name)->IsAbsent()) { logger->Emit(LogLevel::Info, "Base root is concrete, will manage build locally."); return std::nullopt; } if (analyse_context->serve == nullptr) { logger->Emit(LogLevel::Warning, "Cannot treat a root absent without serve"); return std::nullopt; } auto target = id.target.GetNamedTarget(); auto target_root_id = analyse_context->repo_config->TargetRoot(repo_name)->GetAbsentTreeId(); if (not target_root_id) { logger->Emit(LogLevel::Warning, "Failed to get the target root id for repository {}", nlohmann::json(repo_name).dump()); return std::nullopt; } std::filesystem::path module{id.target.ToModule().module}; auto vars = analyse_context->serve->ServeTargetVariables( *target_root_id, (module / *(analyse_context->repo_config->TargetFileName(repo_name))) .string(), target.name); if (not vars) { logger->Emit(LogLevel::Warning, "Failed to obtain variables for {}", id.target.ToString()); return std::nullopt; } auto effective_config = id.config.Prune(*vars); logger->Emit(LogLevel::Info, "Effective configuration {}", effective_config.ToString()); auto repo_key = analyse_context->repo_config->RepositoryKey( *analyse_context->storage, target.repository); if (not repo_key) { logger->Emit(LogLevel::Warning, "Cannot obtain repository key"); return std::nullopt; } auto target_cache_key = analyse_context->storage->TargetCache().ComputeKey( *repo_key, target, effective_config); if (not target_cache_key) { logger->Emit(LogLevel::Warning, "Failed to obtain target-cache key"); return std::nullopt; } logger->Emit(LogLevel::Info, "Target cache key {}", target_cache_key->Id().ToString()); auto res = analyse_context->serve->ServeTarget( *target_cache_key, *repo_key, /*keep_artifacts_root=*/true); if (not res) { logger->Emit(LogLevel::Warning, "Could not obtain target from serve"); return std::nullopt; } if (res->index() != 3) { logger->Emit(LogLevel::Warning, "Failed to obtain root from serve"); return std::nullopt; } auto target_cache_value = std::get<3>(*res); auto const& [entry, info] = target_cache_value; auto result = entry.ToResult(); if (not result) { logger->Emit(LogLevel::Warning, "Reading entry cache entry failed."); return std::nullopt; } auto wrapped_logger = std::make_shared( [&logger](auto const& msg, bool fatal) { logger->Emit(fatal ? LogLevel::Warning : LogLevel::Info, "While computing root from stage:{}", msg); }); auto git_tree = ArtifactsRoot( result->artifact_stage, wrapped_logger, /*rehash=*/std::nullopt); if (not git_tree) { logger->Emit( LogLevel::Warning, "Failed to compute git tree from obtained artifact stage {}", result->artifact_stage->ToString()); return std::nullopt; } logger->Emit(LogLevel::Info, "Tree identifier for root is {}.", *git_tree); return git_tree; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/inquire_serve.hpp000066400000000000000000000024171516554100600310460ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_INQUIRE_SERVE_HPP #define INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_INQUIRE_SERVE_HPP #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/analyse_context.hpp" // Inquire serve for a given target and report the artifact stage as git tree // identifier. [[nodiscard]] auto InquireServe( gsl::not_null const& analyse_context, BuildMaps::Target::ConfiguredTarget const& id, gsl::not_null const& logger) -> std::optional; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/lookup_cache.cpp000066400000000000000000000131741516554100600306170ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/lookup_cache.hpp" #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/computed_roots/artifacts_root.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget, gsl::not_null const& repository_config, Storage const& storage, AsyncMapConsumerLoggerPtr const& logger, std::optional const& rehash) -> expected, std::monostate> { auto const* target_root = repository_config->TargetRoot(ctarget.target.ToModule().repository); if ((target_root == nullptr) or target_root->IsAbsent()) { return expected, std::monostate>( std::nullopt); } auto repo_key = repository_config->RepositoryKey( storage, ctarget.target.GetNamedTarget().repository); if (not repo_key) { (*logger)(fmt::format("Repository {} is not content-fixed", nlohmann::json( ctarget.target.GetNamedTarget().repository)), /*fatal=*/true); return unexpected(std::monostate{}); } auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(repository_config, 1); nlohmann::json targets_file{}; bool failed{false}; { TaskSystem ts{1}; targets_file_map.ConsumeAfterKeysReady( &ts, {ctarget.target.ToModule()}, [&targets_file](auto values) { targets_file = *values[0]; }, [&logger, &failed](auto const& msg, bool fatal) { (*logger)( fmt::format("While searching for target description:\n{}", msg), fatal); failed = failed or fatal; }); } if (failed) { return unexpected(std::monostate{}); } auto desc_it = targets_file.find(ctarget.target.GetNamedTarget().name); if (desc_it == targets_file.end()) { (*logger)("Not referring to a defined target", /*fatal=*/true); return unexpected(std::monostate{}); } nlohmann::json const& desc = *desc_it; auto rule_it = desc.find("type"); if (rule_it == desc.end()) { (*logger)(fmt::format("No type specified in target-description {}", desc.dump()), true); return unexpected(std::monostate{}); } auto const& rule = *rule_it; if (not(rule.is_string() and rule.get() == "export")) { (*logger)(fmt::format("Target not an export target, but of type {}", rule.dump()), true); return unexpected(std::monostate{}); } auto reader = BuildMaps::Base::FieldReader::CreatePtr( desc, ctarget.target, "export target", logger); auto flexible_vars = reader->ReadStringList("flexible_config"); if (not flexible_vars) { return unexpected(std::monostate{}); } auto effective_config = ctarget.config.Prune(*flexible_vars); auto cache_key = storage.TargetCache().ComputeKey( *repo_key, ctarget.target.GetNamedTarget(), effective_config); if (not cache_key) { (*logger)("Target-cache key generation failed", true); return unexpected(std::monostate{}); } auto target_cache_value = storage.TargetCache().Read(*cache_key); if (not target_cache_value) { return expected, std::monostate>( std::nullopt); } auto const& [entry, info] = *target_cache_value; auto result = entry.ToResult(); if (not result) { (*logger)(fmt::format("Failed to deserialize cache entry {} for key {}", info.ToString(), cache_key->Id().ToString()), true); return unexpected(std::monostate{}); } auto wrapped_logger = std::make_shared( [&logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While computing git tree for artifacts stage:\n{}", msg), fatal); }); auto root = ArtifactsRoot(result->artifact_stage, wrapped_logger, rehash); if (not root) { return unexpected(std::monostate{}); } return root; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/lookup_cache.hpp000066400000000000000000000027431516554100600306240ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP #define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" auto LookupCache( BuildMaps::Target::ConfiguredTarget const& ctarget, gsl::not_null const& repository_config, Storage const& storage, AsyncMapConsumerLoggerPtr const& logger, std::optional const& rehash = std::nullopt) -> expected, std::monostate>; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/roots_progress.cpp000066400000000000000000000035511516554100600312530ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/computed_roots/roots_progress.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" auto RootsProgress::Reporter(gsl::not_null const& stats, gsl::not_null const& tasks, Logger const* logger) noexcept -> progress_reporter_t { return BaseProgressReporter::Reporter([stats, tasks, logger]() { auto const& sample = tasks->Sample(); // Note: order matters; queued has to be queried last int cached = stats->ActionsCachedCounter(); int run = stats->ActionsExecutedCounter(); int served = stats->ExportsServedCounter(); int queued = stats->ActionsQueuedCounter(); int active = queued - run - cached - served; auto now_msg = std::string{}; if (active > 0 and not sample.empty()) { now_msg = fmt::format(" ({}{})", sample, active > 1 ? ", ..." : ""); } Logger::Log( logger, LogLevel::Progress, "Computed Roots: {} cached, {} served, {} run, {} processing{}.", cached, served, run, active, now_msg); }); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/computed_roots/roots_progress.hpp000066400000000000000000000025241516554100600312570ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_ROOTS_REPORTER_HPP #define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_ROOTS_REPORTER_HPP #include "gsl/gsl" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" /// \brief Reporter for progress while computing roots class RootsProgress { public: [[nodiscard]] static auto Reporter(gsl::not_null const& stats, gsl::not_null const& tasks, Logger const* logger = nullptr) noexcept -> progress_reporter_t; }; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/000077500000000000000000000000001516554100600237235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/TARGETS000066400000000000000000000023701516554100600247610ustar00rootroot00000000000000{ "hasher": { "type": ["@", "rules", "CC", "library"] , "name": ["hasher"] , "hdrs": ["hasher.hpp"] , "srcs": ["hasher.cpp"] , "stage": ["src", "buildtool", "crypto"] , "deps": [["src/utils/cpp", "hex_string"]] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["@", "ssl", "", "crypto"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "hash_function": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_function"] , "hdrs": ["hash_function.hpp"] , "srcs": ["hash_function.cpp"] , "deps": ["hasher", ["@", "gsl", "", "gsl"]] , "private-deps": [ ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] ] , "stage": ["src", "buildtool", "crypto"] } , "hash_info": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_info"] , "hdrs": ["hash_info.hpp"] , "srcs": ["hash_info.cpp"] , "deps": ["hash_function", ["src/utils/cpp", "expected"]] , "private-deps": [ "hasher" , ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "protocol_traits"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "crypto"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hash_function.cpp000066400000000000000000000076061516554100600272700ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hash_function.hpp" #include #include #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/incremental_reader.hpp" namespace { [[nodiscard]] auto CreateGitTreeTag(std::size_t size) noexcept -> std::string { return std::string("tree ") + std::to_string(size) + '\0'; } [[nodiscard]] auto CreateGitBlobTag(std::size_t size) noexcept -> std::string { return std::string("blob ") + std::to_string(size) + '\0'; } } // namespace auto HashFunction::HashBlobData(std::string const& data) const noexcept -> Hasher::HashDigest { return HashTaggedLine(data, CreateGitBlobTag); } auto HashFunction::HashTreeData(std::string const& data) const noexcept -> Hasher::HashDigest { return HashTaggedLine(data, CreateGitTreeTag); } auto HashFunction::PlainHashData(std::string const& data) const noexcept -> Hasher::HashDigest { return HashTaggedLine(data, std::nullopt); } auto HashFunction::HashTaggedLine(std::string const& data, std::optional tag_creator) const noexcept -> Hasher::HashDigest { auto hasher = MakeHasher(); if (type_ == Type::GitSHA1 and tag_creator.has_value()) { hasher.Update(std::invoke(*tag_creator, data.size())); } hasher.Update(data); return std::move(hasher).Finalize(); } auto HashFunction::HashBlobFile(std::filesystem::path const& path) const noexcept -> std::optional> { return HashTaggedFile(path, CreateGitBlobTag); } auto HashFunction::HashTreeFile(std::filesystem::path const& path) const noexcept -> std::optional> { return HashTaggedFile(path, CreateGitTreeTag); } auto HashFunction::HashTaggedFile(std::filesystem::path const& path, TagCreator const& tag_creator) const noexcept -> std::optional> { auto const size = std::filesystem::file_size(path); static constexpr std::size_t kChunkSize{4048}; auto hasher = MakeHasher(); if (type_ == Type::GitSHA1) { hasher.Update(std::invoke(tag_creator, size)); } auto const to_read = IncrementalReader::FromFile(kChunkSize, path); if (not to_read.has_value()) { Logger::Log(LogLevel::Debug, "Failed to create a reader for {}: {}", path.string(), to_read.error()); return std::nullopt; } try { for (auto chunk : *to_read) { if (not chunk.has_value()) { Logger::Log(LogLevel::Debug, "Error while trying to hash {}: {}", path.string(), chunk.error()); return std::nullopt; } hasher.Update(std::string{*chunk}); } } catch (std::exception const& e) { Logger::Log(LogLevel::Debug, "Error while trying to hash {}: {}", path.string(), e.what()); return std::nullopt; } return std::make_pair(std::move(hasher).Finalize(), size); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hash_function.hpp000066400000000000000000000076461516554100600273010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP #define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/crypto/hasher.hpp" /// \brief Hash function used for the entire buildtool. class HashFunction { public: enum class Type : std::uint8_t { GitSHA1, ///< SHA1 for plain hashes, and Git for blobs and trees. PlainSHA256 ///< SHA256 for all hashes. }; explicit HashFunction(Type type) noexcept : type_{type} { static_assert( sizeof(HashFunction) <= sizeof(void*), "HashFunction is passed and stored by value. If the " "class is extended so that its size exceeds the size of a pointer, " "the way how HashFunction is passed and stored must be changed."); } [[nodiscard]] auto GetType() const noexcept -> Type { return type_; } /// \brief Compute the blob hash of a string. [[nodiscard]] auto HashBlobData(std::string const& data) const noexcept -> Hasher::HashDigest; /// \brief Compute the tree hash of a string. [[nodiscard]] auto HashTreeData(std::string const& data) const noexcept -> Hasher::HashDigest; /// \brief Compute the plain hash of a string. [[nodiscard]] auto PlainHashData(std::string const& data) const noexcept -> Hasher::HashDigest; /// \brief Compute the blob hash of a file or std::nullopt on IO error. [[nodiscard]] auto HashBlobFile( std::filesystem::path const& path) const noexcept -> std::optional>; /// \brief Compute the tree hash of a file or std::nullopt on IO error. [[nodiscard]] auto HashTreeFile( std::filesystem::path const& path) const noexcept -> std::optional>; /// \brief Obtain incremental hasher for computing plain hashes. [[nodiscard]] auto MakeHasher() const noexcept -> Hasher { std::optional hasher; switch (type_) { case Type::GitSHA1: hasher = Hasher::Create(Hasher::HashType::SHA1); break; case Type::PlainSHA256: hasher = Hasher::Create(Hasher::HashType::SHA256); break; } Ensures(hasher.has_value()); return *std::move(hasher); } private: Type const type_; using TagCreator = std::function; [[nodiscard]] auto HashTaggedLine(std::string const& data, std::optional tag_creator) const noexcept -> Hasher::HashDigest; [[nodiscard]] auto HashTaggedFile( std::filesystem::path const& path, TagCreator const& tag_creator) const noexcept -> std::optional>; }; [[nodiscard]] constexpr auto ToString(HashFunction::Type type) noexcept -> const char* { switch (type) { case HashFunction::Type::GitSHA1: return "git-SHA1"; case HashFunction::Type::PlainSHA256: return "plain-SHA256"; } Ensures(false); // unreachable } #endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hash_info.cpp000066400000000000000000000072331516554100600263720ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hash_info.hpp" #include "fmt/core.h" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { inline constexpr auto kSHA1EmptyGitBlobHash = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"; } // namespace HashInfo::HashInfo() noexcept : hash_{kSHA1EmptyGitBlobHash}, hash_type_{HashFunction::Type::GitSHA1}, is_tree_{false} {} auto HashInfo::Create(HashFunction::Type type, std::string hash, bool is_tree) noexcept -> expected { if (auto error = HashInfo::ValidateInput(type, hash, is_tree)) { return unexpected{*std::move(error)}; } return HashInfo(std::move(hash), type, is_tree); } auto HashInfo::HashData(HashFunction hash_function, std::string const& content, bool is_tree) noexcept -> HashInfo { auto const hash_digest = is_tree ? hash_function.HashTreeData(content) : hash_function.HashBlobData(content); return HashInfo{ hash_digest.HexString(), hash_function.GetType(), is_tree and ProtocolTraits::IsTreeAllowed(hash_function.GetType())}; } auto HashInfo::HashFile(HashFunction hash_function, std::filesystem::path const& path, bool is_tree) noexcept -> std::optional> { auto const hash_digest = is_tree ? hash_function.HashTreeFile(path) : hash_function.HashBlobFile(path); if (not hash_digest) { return std::nullopt; } return std::pair{HashInfo{hash_digest->first.HexString(), hash_function.GetType(), is_tree and hash_function.GetType() == HashFunction::Type::GitSHA1}, hash_digest->second}; } auto HashInfo::operator==(HashInfo const& other) const noexcept -> bool { return hash_ == other.hash_ and is_tree_ == other.is_tree_; } auto HashInfo::ValidateInput(HashFunction::Type type, std::string const& hash, bool is_tree) noexcept -> std::optional { if (is_tree and not ProtocolTraits::IsTreeAllowed(type)) { return fmt::format( "HashInfo: hash {} is expected to be {}.\nTrees are " "not allowed in this mode.", hash, ToString(type)); } if (auto const exp_size = HashFunction{type}.MakeHasher().GetHashLength(); hash.size() != exp_size) { return fmt::format( "HashInfo: hash {} is expected to be {}.\n It must have a length " "of {}, but its length is {}.", hash, ToString(type), exp_size, hash.size()); } if (not IsHexString(hash)) { return fmt::format("HashInfo: Invalid hash {}", hash); } return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hash_info.hpp000066400000000000000000000103631516554100600263750ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_INFO_HPP #define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_INFO_HPP #include #include #include #include #include #include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/expected.hpp" /// \brief A collection of data related to a specific hash. Once it is /// constructed, it holds a valid hexadecimal (always unprefixed) hash with some /// additional information about the method of hashing. class HashInfo final { public: explicit HashInfo() noexcept; /// \brief Build HashInfo based on 'external' data that cannot be trusted. A /// number of validation checks is happening /// \param type Type of the hash function used to create the hash /// \param hash A hexadecimal hash /// \param is_tree Tree or blob. Note that trees are not allowed in the /// compatible mode. /// \return Validated HashInfo on success or an error message on failure. [[nodiscard]] static auto Create(HashFunction::Type type, std::string hash, bool is_tree) noexcept -> expected; /// \brief Hash content and build HashInfo /// \param hash_function Hash function to be used /// \param content Content to be hashed /// \param is_tree Tree or blob, the type of the algorithm to be used for /// hashing. Note that HashInfo may return another value from IsTree in /// compatible mode. [[nodiscard]] static auto HashData(HashFunction hash_function, std::string const& content, bool is_tree) noexcept -> HashInfo; /// \brief Hash file and build HashInfo /// \param hash_function Hash function to be use /// \param path File to be hashed /// \param is_tree Tree or blob, the type of the algorithm to be used for /// hashing. Note that HashInfo may return another value from IsTree in /// compatible mode. /// \return A combination of the hash of the file and file's size or /// std::nullopt on IO failure. [[nodiscard]] static auto HashFile(HashFunction hash_function, std::filesystem::path const& path, bool is_tree) noexcept -> std::optional>; [[nodiscard]] auto Hash() const& noexcept -> std::string const& { return hash_; } [[nodiscard]] auto Hash() && -> std::string { return std::move(hash_); } [[nodiscard]] auto HashType() const noexcept -> HashFunction::Type { return hash_type_; } [[nodiscard]] auto IsTree() const noexcept -> bool { return is_tree_; } [[nodiscard]] auto operator==(HashInfo const& other) const noexcept -> bool; private: std::string hash_; HashFunction::Type hash_type_; /// \brief Tree or blob algorithm was used for hashing. is_tree_ can be true /// in the native mode only, in compatible it falls back to false during /// hashing via HashData/HashFile or an error occurs during validation. bool is_tree_; explicit HashInfo(std::string hash, HashFunction::Type type, bool is_tree) noexcept : hash_{std::move(hash)}, hash_type_{type}, is_tree_{is_tree} {} [[nodiscard]] static auto ValidateInput(HashFunction::Type type, std::string const& hash, bool is_tree) noexcept -> std::optional; }; #endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_INFO_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hasher.cpp000066400000000000000000000160611516554100600257050ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hasher.hpp" #include #include #include #include #include #include "gsl/gsl" #include "openssl/sha.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" // Since it is impossible either forward declare SHA*_CTX (they are typedefs), // nor their basic classes (they are called differently in OpenSSL and // BoringSSL), an intermediate struct is forward declared in the header. using VariantContext = std::variant; struct Hasher::ShaContext final : VariantContext { using VariantContext::VariantContext; }; namespace { inline constexpr int kOpenSslTrue = 1; [[nodiscard]] auto CreateShaContext(Hasher::HashType type) noexcept -> std::unique_ptr { switch (type) { case Hasher::HashType::SHA1: return std::make_unique(SHA_CTX{}); case Hasher::HashType::SHA256: return std::make_unique(SHA256_CTX{}); case Hasher::HashType::SHA512: return std::make_unique(SHA512_CTX{}); } return nullptr; // make gcc happy } template [[nodiscard]] auto Visit(gsl::not_null const& ctx, Args&&... visitor_args) noexcept { try { return std::visit(TVisitor{std::forward(visitor_args)...}, *ctx); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "HashFunction::{} failed with an exception:\n{}", TVisitor::kLogInfo, e.what()); Ensures(false); } } struct InitializeVisitor final { static constexpr std::string_view kLogInfo = "Initialize"; // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA_CTX& ctx) const -> bool { return SHA1_Init(&ctx) == kOpenSslTrue; } // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA256_CTX& ctx) const -> bool { return SHA256_Init(&ctx) == kOpenSslTrue; } // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA512_CTX& ctx) const -> bool { return SHA512_Init(&ctx) == kOpenSslTrue; } }; struct UpdateVisitor final { static constexpr std::string_view kLogInfo = "Update"; explicit UpdateVisitor(gsl::not_null const& data) : data_{*data} {} // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA_CTX& ctx) const -> bool { return SHA1_Update(&ctx, data_.data(), data_.size()) == kOpenSslTrue; } // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA256_CTX& ctx) const -> bool { return SHA256_Update(&ctx, data_.data(), data_.size()) == kOpenSslTrue; } // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] auto operator()(SHA512_CTX& ctx) const -> bool { return SHA512_Update(&ctx, data_.data(), data_.size()) == kOpenSslTrue; } private: std::string const& data_; }; struct FinalizeVisitor final { static constexpr std::string_view kLogInfo = "Finalize"; // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] inline auto operator()(SHA_CTX& ctx) const -> std::optional; // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] inline auto operator()(SHA256_CTX& ctx) const -> std::optional; // NOLINTNEXTLINE(google-runtime-references) [[nodiscard]] inline auto operator()(SHA512_CTX& ctx) const -> std::optional; }; struct LengthVisitor final { static constexpr std::string_view kLogInfo = "GetHashLength"; [[nodiscard]] constexpr auto operator()(SHA_CTX const& /*unused*/) const -> std::size_t { return SHA_DIGEST_LENGTH * kCharsPerNumber; } [[nodiscard]] constexpr auto operator()(SHA256_CTX const& /*unused*/) const -> std::size_t { return SHA256_DIGEST_LENGTH * kCharsPerNumber; } [[nodiscard]] constexpr auto operator()(SHA512_CTX const& /*unused*/) const -> std::size_t { return SHA512_DIGEST_LENGTH * kCharsPerNumber; } private: static constexpr size_t kCharsPerNumber = std::numeric_limits::max() / std::numeric_limits::max(); }; } // namespace Hasher::Hasher(std::unique_ptr sha_ctx) noexcept : sha_ctx_{std::move(sha_ctx)} {} // Explicitly declared and then defaulted dtor and move ctor/operator are needed // to compile std::unique_ptr of an incomplete type. Hasher::Hasher(Hasher&& other) noexcept = default; auto Hasher::operator=(Hasher&& other) noexcept -> Hasher& = default; Hasher::~Hasher() noexcept = default; auto Hasher::Create(HashType type) noexcept -> std::optional { auto sha_ctx = CreateShaContext(type); if (sha_ctx != nullptr and Visit(sha_ctx.get())) { return std::optional{Hasher{std::move(sha_ctx)}}; } return std::nullopt; } auto Hasher::Update(std::string const& data) noexcept -> bool { return Visit(sha_ctx_.get(), &data); } auto Hasher::Finalize() && noexcept -> HashDigest { if (auto hash = Visit(sha_ctx_.get())) { return HashDigest{std::move(*hash)}; } Logger::Log(LogLevel::Error, "Failed to compute hash."); Ensures(false); } auto Hasher::GetHashLength() const noexcept -> std::size_t { return Visit(sha_ctx_.get()); } namespace { auto FinalizeVisitor::operator()(SHA_CTX& ctx) const -> std::optional { auto out = std::array{}; if (SHA1_Final(out.data(), &ctx) == kOpenSslTrue) { return std::string{out.begin(), out.end()}; } return std::nullopt; } auto FinalizeVisitor::operator()(SHA256_CTX& ctx) const -> std::optional { auto out = std::array{}; if (SHA256_Final(out.data(), &ctx) == kOpenSslTrue) { return std::string{out.begin(), out.end()}; } return std::nullopt; } auto FinalizeVisitor::operator()(SHA512_CTX& ctx) const -> std::optional { auto out = std::array{}; if (SHA512_Final(out.data(), &ctx) == kOpenSslTrue) { return std::string{out.begin(), out.end()}; } return std::nullopt; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/crypto/hasher.hpp000066400000000000000000000061161516554100600257120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASHER_HPP #define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASHER_HPP #include #include #include #include #include #include // std::move #include "src/utils/cpp/hex_string.hpp" /// \brief Incremental hasher. class Hasher final { public: /// \brief Types of hash implementations supported by generator. enum class HashType : std::uint8_t { SHA1, SHA256, SHA512 }; struct ShaContext; /// \brief The universal hash digest. /// The type of hash and the digest length depends on the hash /// implementation used to generated this digest. class HashDigest final { friend Hasher; public: /// \brief Get pointer to raw bytes of digest. /// Length can be obtained using \ref Length. [[nodiscard]] auto Bytes() const& noexcept -> std::string const& { return bytes_; } [[nodiscard]] auto Bytes() && noexcept -> std::string { return std::move(bytes_); } /// \brief Get hexadecimal string of digest. /// Length is twice the length of raw bytes (\ref Length). [[nodiscard]] auto HexString() const -> std::string { return ToHexString(bytes_); } /// \brief Get digest length in raw bytes. [[nodiscard]] auto Length() const -> std::size_t { return bytes_.size(); } private: std::string bytes_; explicit HashDigest(std::string bytes) : bytes_{std::move(bytes)} {} }; /// \brief Create and initialize a hasher /// \return An initialized hasher on success or std::nullopt on failure. [[nodiscard]] static auto Create(HashType type) noexcept -> std::optional; Hasher(Hasher&& other) noexcept; auto operator=(Hasher&& other) noexcept -> Hasher&; Hasher(Hasher const& other) noexcept = delete; auto operator=(Hasher const& other) noexcept -> Hasher& = delete; ~Hasher() noexcept; /// \brief Feed data to the hasher. auto Update(std::string const& data) noexcept -> bool; /// \brief Finalize hash. [[nodiscard]] auto Finalize() && noexcept -> HashDigest; /// \brief Obtain length of the resulting hash string. [[nodiscard]] auto GetHashLength() const noexcept -> std::size_t; private: std::unique_ptr sha_ctx_; explicit Hasher(std::unique_ptr sha_ctx) noexcept; }; #endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASHER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/000077500000000000000000000000001516554100600252375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/000077500000000000000000000000001516554100600272025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/TARGETS000066400000000000000000000033041516554100600302360ustar00rootroot00000000000000{ "execution_config": { "type": ["@", "rules", "CC", "library"] , "name": ["execution_config"] , "hdrs": ["execution_config.hpp"] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] } , "bazel_msg_factory": { "type": ["@", "rules", "CC", "library"] , "name": ["bazel_msg_factory"] , "hdrs": ["bazel_msg_factory.hpp"] , "srcs": ["bazel_msg_factory.cpp"] , "deps": [ "directory_tree" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path"] ] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] } , "directory_tree": { "type": ["@", "rules", "CC", "library"] , "name": ["directory_tree"] , "hdrs": ["directory_tree.hpp"] , "srcs": ["directory_tree.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_engine/dag", "dag"] ] , "private-deps": [ ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "gsl"] ] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp000066400000000000000000001136431516554100600334100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path.hpp" namespace { /// \brief Serialize protobuf message to string. template [[nodiscard]] auto SerializeMessage(T const& message) noexcept -> std::optional { try { std::string content(message.ByteSizeLong(), '\0'); message.SerializeToArray(content.data(), gsl::narrow(content.size())); return content; } catch (...) { return std::nullopt; } } /// \brief Create protobuf message 'Platform'. [[nodiscard]] auto CreatePlatform( std::vector const& props) noexcept -> std::unique_ptr { auto platform = std::make_unique(); std::copy(props.cbegin(), props.cend(), pb::back_inserter(platform->mutable_properties())); return platform; } /// \brief Create protobuf message 'Directory'. [[nodiscard]] auto CreateDirectory( std::vector const& files, std::vector const& dirs, std::vector const& links) noexcept -> bazel_re::Directory { bazel_re::Directory dir{}; auto copy_nodes = [](auto* pb_container, auto const& nodes) { pb_container->Reserve(gsl::narrow(nodes.size())); std::copy(nodes.begin(), nodes.end(), pb::back_inserter(pb_container)); std::sort( pb_container->begin(), pb_container->end(), [](auto const& l, auto const& r) { return l.name() < r.name(); }); }; copy_nodes(dir.mutable_files(), files); copy_nodes(dir.mutable_directories(), dirs); copy_nodes(dir.mutable_symlinks(), links); return dir; } /// \brief Create protobuf message 'FileNode'. [[nodiscard]] auto CreateFileNode(std::string const& file_name, ObjectType type, ArtifactDigest const& digest) noexcept -> bazel_re::FileNode { bazel_re::FileNode node; node.set_name(file_name); node.set_is_executable(IsExecutableObject(type)); (*node.mutable_digest()) = ArtifactDigestFactory::ToBazel(digest); return node; } /// \brief Create protobuf message 'DirectoryNode'. [[nodiscard]] auto CreateDirectoryNode(std::string const& dir_name, ArtifactDigest const& digest) noexcept -> bazel_re::DirectoryNode { bazel_re::DirectoryNode node; node.set_name(dir_name); (*node.mutable_digest()) = ArtifactDigestFactory::ToBazel(digest); return node; } /// \brief Create protobuf message 'SymlinkNode'. [[nodiscard]] auto CreateSymlinkNode(std::string const& link_name, std::string const& target) noexcept -> bazel_re::SymlinkNode { bazel_re::SymlinkNode node; node.set_name(link_name); node.set_target(target); return node; } /// \brief Create protobuf message SymlinkNode from Digest for multiple /// instances at once [[nodiscard]] auto CreateSymlinkNodesFromDigests( std::vector const& symlink_names, std::vector const& symlink_digests, BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) -> std::optional> { if (symlink_names.size() != symlink_digests.size()) { return std::nullopt; } std::vector symlink_targets; symlink_targets.reserve(symlink_digests.size()); resolve_links(symlink_digests, &symlink_targets); // Fail if the number of resolved symlinks does not match the requested // number. if (symlink_targets.size() != symlink_names.size()) { return std::nullopt; } std::vector symlink_nodes; symlink_nodes.reserve(symlink_targets.size()); for (std::size_t i = 0; i < symlink_targets.size(); ++i) { symlink_nodes.emplace_back( CreateSymlinkNode(symlink_names[i], symlink_targets[i])); } return symlink_nodes; } struct DirectoryNodeBundle final { bazel_re::DirectoryNode message; ArtifactBlob blob; }; /// \brief Create bundle for protobuf message DirectoryNode from Directory. [[nodiscard]] auto CreateDirectoryNodeBundle(std::string const& dir_name, bazel_re::Directory const& dir) -> std::optional { auto content = SerializeMessage(dir); if (not content) { return std::nullopt; } // SHA256 is used since bazel types are processed here. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; auto blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, *std::move(content)); if (not blob.has_value()) { return std::nullopt; } auto const digest = blob->GetDigest(); return DirectoryNodeBundle{.message = CreateDirectoryNode(dir_name, digest), .blob = *std::move(blob)}; } /// \brief Create bundle for protobuf message Command from args strings. [[nodiscard]] auto CreateCommandBundle( BazelMsgFactory::ActionDigestRequest const& request) -> std::optional { bazel_re::Command msg; // DEPRECATED as of v2.2: platform properties are now specified // directly in the action. See documentation note in the // [Action][build.bazel.remote.execution.v2.Action] for migration. // (https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L646) msg.set_allocated_platform(CreatePlatform(*request.properties).release()); msg.set_working_directory(*request.cwd); std::copy(request.command_line->begin(), request.command_line->end(), pb::back_inserter(msg.mutable_arguments())); // DEPRECATED as of v2.1: use output_paths instead, combining // output_files and output_dirs. std::copy(request.output_files->begin(), request.output_files->end(), pb::back_inserter(msg.mutable_output_files())); std::copy(request.output_dirs->begin(), request.output_dirs->end(), pb::back_inserter(msg.mutable_output_directories())); // NEW in v2.2 std::copy(request.output_paths->begin(), request.output_paths->end(), pb::back_inserter(msg.mutable_output_paths())); std::copy(request.env_vars->begin(), request.env_vars->end(), pb::back_inserter(msg.mutable_environment_variables())); auto content = SerializeMessage(msg); if (not content) { return std::nullopt; } auto blob = ArtifactBlob::FromMemory( request.hash_function, ObjectType::File, *std::move(content)); if (not blob.has_value()) { return std::nullopt; } return *std::move(blob); } /// \brief Create bundle for protobuf message Action from Command. [[nodiscard]] auto CreateActionBundle( ArtifactDigest const& command, BazelMsgFactory::ActionDigestRequest const& request) -> std::optional { using seconds = std::chrono::seconds; using nanoseconds = std::chrono::nanoseconds; auto sec = std::chrono::duration_cast(request.timeout); auto nanos = std::chrono::duration_cast(request.timeout - sec); auto duration = std::make_unique(); duration->set_seconds(sec.count()); duration->set_nanos(static_cast(nanos.count())); bazel_re::Action msg; msg.set_do_not_cache(request.skip_action_cache); msg.set_allocated_timeout(duration.release()); *msg.mutable_command_digest() = ArtifactDigestFactory::ToBazel(command); *msg.mutable_input_root_digest() = ArtifactDigestFactory::ToBazel(*request.exec_dir); // New in version 2.2: clients SHOULD set these platform properties // as well as those in the // [Command][build.bazel.remote.execution.v2.Command]. Servers // SHOULD prefer those set here. // (https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L516) msg.set_allocated_platform(CreatePlatform(*request.properties).release()); auto content = SerializeMessage(msg); if (not content) { return std::nullopt; } auto blob = ArtifactBlob::FromMemory( request.hash_function, ObjectType::File, *std::move(content)); if (not blob.has_value()) { return std::nullopt; } return *std::move(blob); } /// \brief Convert `DirectoryTree` to `DirectoryNodeBundle`. [[nodiscard]] auto DirectoryTreeToBundle( std::string const& root_name, DirectoryTreePtr const& tree, BazelMsgFactory::LinkDigestResolveFunc const& resolve_links, BazelMsgFactory::BlobProcessFunc const& process_blob, std::filesystem::path const& parent = "") noexcept -> std::optional { std::vector file_nodes{}; std::vector dir_nodes{}; std::vector symlink_names{}; std::vector symlink_digests{}; try { for (auto const& [name, node] : *tree) { if (std::holds_alternative(node)) { auto const& dir = std::get(node); auto dir_bundle = DirectoryTreeToBundle( name, dir, resolve_links, process_blob, parent / name); if (not dir_bundle) { return std::nullopt; } dir_nodes.emplace_back(std::move(dir_bundle->message)); if (not process_blob(std::move(dir_bundle->blob))) { return std::nullopt; } } else { auto const& artifact = std::get(node); auto const& object_info = artifact->Info(); if (not object_info) { return std::nullopt; } if (IsTreeObject(object_info->type)) { dir_nodes.emplace_back( CreateDirectoryNode(name, object_info->digest)); } else if (IsSymlinkObject(object_info->type)) { // for symlinks we need to retrieve the data from the // digest, which we will handle in bulk symlink_names.emplace_back(name); symlink_digests.emplace_back(object_info->digest); } else { file_nodes.emplace_back(CreateFileNode( name, object_info->type, object_info->digest)); } } } auto symlink_nodes = CreateSymlinkNodesFromDigests( symlink_names, symlink_digests, resolve_links); if (not symlink_nodes.has_value()) { return std::nullopt; } return CreateDirectoryNodeBundle( root_name, CreateDirectory(file_nodes, dir_nodes, *std::move(symlink_nodes))); } catch (...) { return std::nullopt; } return std::nullopt; } [[nodiscard]] auto GetContentFromGitEntry( BazelMsgFactory::GitReadFunc const& read_git, ArtifactDigest const& digest, ObjectType entry_type) -> expected { auto read_git_res = read_git(digest, entry_type); if (not read_git_res) { return unexpected{ fmt::format("failed reading Git entry {}", digest.hash())}; } if (std::holds_alternative(read_git_res.value())) { return std::get(std::move(read_git_res).value()); } if (std::holds_alternative(read_git_res.value())) { auto content = FileSystemManager::ReadFile( std::get(std::move(read_git_res).value())); if (not content) { return unexpected{fmt::format("failed reading content of tree {}", digest.hash())}; } return *std::move(content); } return unexpected{ fmt::format("unexpected failure reading Git entry {}", digest.hash())}; } } // namespace auto BazelMsgFactory::CreateDirectoryDigestFromTree( DirectoryTreePtr const& tree, LinkDigestResolveFunc const& resolve_links, BlobProcessFunc const& process_blob) noexcept -> std::optional { auto bundle = DirectoryTreeToBundle("", tree, resolve_links, process_blob); if (not bundle) { return std::nullopt; } auto digest = bundle->blob.GetDigest(); try { if (not process_blob(std::move(bundle->blob))) { return std::nullopt; } } catch (...) { return std::nullopt; } return digest; } auto BazelMsgFactory::CreateDirectoryDigestFromGitTree( ArtifactDigest const& digest, GitReadFunc const& read_git, BlobStoreFunc const& store_file, TreeStoreFunc const& store_dir, SymlinkStoreFunc const& store_symlink, RehashedDigestReadFunc const& read_rehashed, RehashedDigestStoreFunc const& store_rehashed) noexcept -> expected { std::vector files{}; std::vector dirs{}; std::vector symlinks{}; try { // read tree object auto const tree_content = GetContentFromGitEntry(read_git, digest, ObjectType::Tree); if (not tree_content) { return unexpected{tree_content.error()}; } // Git-SHA1 hashing is used for reading from git HashFunction const hash_function{HashFunction::Type::GitSHA1}; // the tree digest is in native mode, so no need for rehashing content auto skip_symlinks = [](auto const& /*unused*/) { return true; }; auto const entries = GitRepo::ReadTreeData( *tree_content, digest.hash(), skip_symlinks, /*is_hex_id=*/true); if (not entries) { return unexpected{fmt::format("failed reading entries of tree {}", digest.hash())}; } // handle tree entries for (auto const& [raw_id, es] : *entries) { auto const hex_id = ToHexString(raw_id); for (auto const& entry : es) { // get native digest of entry auto const git_digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, hex_id, /*size is unknown*/ 0, IsTreeObject(entry.type)); if (not git_digest) { return unexpected{git_digest.error()}; } // get any cached digest mapping, to avoid unnecessary work auto const cached_obj = read_rehashed(*git_digest); if (not cached_obj) { return unexpected{cached_obj.error()}; } // create and store the directory entry switch (entry.type) { case ObjectType::Tree: { if (cached_obj.value()) { // no work to be done if we already know the digest dirs.emplace_back(CreateDirectoryNode( entry.name, cached_obj.value()->digest)); } else { // create and store sub directory auto const dir_digest = CreateDirectoryDigestFromGitTree( *git_digest, read_git, store_file, store_dir, store_symlink, read_rehashed, store_rehashed); if (not dir_digest) { return unexpected{dir_digest.error()}; } dirs.emplace_back( CreateDirectoryNode(entry.name, *dir_digest)); // no need to cache the digest mapping, as this was // done in the recursive call } } break; case ObjectType::Symlink: { // create and store symlink; for this entry type the // cached digest is ignored because we always need the // target (i.e., the symlink content) auto const sym_target = GetContentFromGitEntry( read_git, *git_digest, ObjectType::Symlink); if (not sym_target) { return unexpected{sym_target.error()}; } auto const sym_digest = store_symlink(*sym_target); if (not sym_digest) { return unexpected{fmt::format( "failed storing symlink {}", hex_id)}; } symlinks.emplace_back( CreateSymlinkNode(entry.name, *sym_target)); // while useless for future symlinks, cache digest // mapping for file-type blobs with same content if (auto error_msg = store_rehashed(*git_digest, *sym_digest, ObjectType::Symlink)) { return unexpected{*std::move(error_msg)}; } } break; default: { if (cached_obj.value()) { // no work to be done if we already know the digest files.emplace_back( CreateFileNode(entry.name, entry.type, cached_obj.value()->digest)); } else { // create and store file; here we want to NOT read // the content if from CAS, where we can rehash via // streams! auto const read_git_file = read_git(*git_digest, entry.type); if (not read_git_file) { return unexpected{ fmt::format("failed reading Git entry ")}; } auto const file_digest = store_file( *read_git_file, IsExecutableObject(entry.type)); if (not file_digest) { return unexpected{fmt::format( "failed storing file {}", hex_id)}; } files.emplace_back(CreateFileNode( entry.name, entry.type, *file_digest)); // cache digest mapping if (auto error_msg = store_rehashed( *git_digest, *file_digest, entry.type)) { return unexpected{*std::move(error_msg)}; } } } } } } // create and store tree auto const bytes = SerializeMessage(CreateDirectory(files, dirs, symlinks)); if (not bytes) { return unexpected{ fmt::format("failed serializing bazel Directory for tree {}", digest.hash())}; } auto const tree_digest = store_dir(*bytes); if (not tree_digest) { return unexpected{fmt::format( "failed storing bazel Directory for tree {}", digest.hash())}; } // cache digest mapping if (auto error_msg = store_rehashed(digest, *tree_digest, ObjectType::Tree)) { return unexpected{*std::move(error_msg)}; } // return digest return *tree_digest; } catch (std::exception const& ex) { return unexpected{fmt::format( "creating bazel Directory digest unexpectedly failed with:\n{}", ex.what())}; } } // NOLINTNEXTLINE(misc-no-recursion) auto BazelMsgFactory::CreateGitTreeDigestFromDirectory( ArtifactDigest const& digest, PathReadFunc const& read_path, FileStoreFunc const& store_file, TreeStoreFunc const& store_tree, SymlinkStoreFunc const& store_symlink, RehashedDigestReadFunc const& read_rehashed, RehashedDigestStoreFunc const& store_rehashed) noexcept -> expected { GitRepo::tree_entries_t entries{}; try { // read directory object auto const tree_path = read_path(digest, ObjectType::Tree); if (not tree_path) { return unexpected{ fmt::format("failed reading CAS entry {}", digest.hash())}; } auto const tree_content = FileSystemManager::ReadFile(*tree_path); if (not tree_content) { return unexpected{fmt::format("failed reading content of tree {}", digest.hash())}; } auto dir = BazelMsgFactory::MessageFromString( *tree_content); // process subdirectories for (auto const& subdir : dir->directories()) { // get digest auto const subdir_digest = ArtifactDigestFactory::FromBazel( HashFunction::Type::PlainSHA256, subdir.digest()); if (not subdir_digest) { return unexpected{subdir_digest.error()}; } // get any cached digest mapping, to avoid unnecessary work auto const cached_obj = read_rehashed(*subdir_digest); if (not cached_obj) { return unexpected{cached_obj.error()}; } if (cached_obj.value()) { // no work to be done, just add to map if (auto raw_id = FromHexString(cached_obj.value()->digest.hash())) { entries[std::move(*raw_id)].emplace_back(subdir.name(), ObjectType::Tree); } else { return unexpected{fmt::format( "failed to get raw id for cached dir digest {}", cached_obj.value()->digest.hash())}; } } else { // recursively get the subdirectory digest auto const tree_digest = CreateGitTreeDigestFromDirectory(*subdir_digest, read_path, store_file, store_tree, store_symlink, read_rehashed, store_rehashed); if (not tree_digest) { return unexpected{tree_digest.error()}; } if (auto raw_id = FromHexString(tree_digest->hash())) { entries[std::move(*raw_id)].emplace_back(subdir.name(), ObjectType::Tree); } else { return unexpected{ fmt::format("failed to get raw id for tree digest {}", tree_digest->hash())}; } // no need to cache the digest mapping, as this was done in the // recursive call } } // process symlinks for (auto const& sym : dir->symlinks()) { // get digest auto const sym_digest = ArtifactDigestFactory::HashDataAs( HashFunction{HashFunction::Type::PlainSHA256}, sym.target()); // get any cached digest mapping, to avoid unnecessary work auto const cached_obj = read_rehashed(sym_digest); if (not cached_obj) { return unexpected{cached_obj.error()}; } if (cached_obj.value()) { // no work to be done, just add to map if (auto raw_id = FromHexString(cached_obj.value()->digest.hash())) { entries[std::move(*raw_id)].emplace_back( sym.name(), ObjectType::Symlink); } else { return unexpected{fmt::format( "failed to get raw id for cached symlink digest {}", cached_obj.value()->digest.hash())}; } } else { // rehash symlink auto const blob_digest = store_symlink(sym.target()); if (not blob_digest) { return unexpected{ fmt::format("failed rehashing as blob symlink {}", sym_digest.hash())}; } if (auto raw_id = FromHexString(blob_digest->hash())) { entries[std::move(*raw_id)].emplace_back( sym.name(), ObjectType::Symlink); } else { return unexpected{fmt::format( "failed to get raw id for symlink blob digest {}", blob_digest->hash())}; } // while useless for future symlinks, cache digest mapping for // file-type blobs with same content if (auto error_msg = store_rehashed( sym_digest, *blob_digest, ObjectType::Symlink)) { return unexpected{*std::move(error_msg)}; } } } // process files for (auto const& file : dir->files()) { // get digest auto const file_digest = ArtifactDigestFactory::FromBazel( HashFunction::Type::PlainSHA256, file.digest()); if (not file_digest) { return unexpected{file_digest.error()}; } auto const file_type = file.is_executable() ? ObjectType::Executable : ObjectType::File; // get any cached digest mapping, to avoid unnecessary work auto const cached_obj = read_rehashed(*file_digest); if (not cached_obj) { return unexpected{cached_obj.error()}; } if (cached_obj.value()) { // no work to be done, just add to map if (auto raw_id = FromHexString(cached_obj.value()->digest.hash())) { entries[std::move(*raw_id)].emplace_back(file.name(), file_type); } else { return unexpected{fmt::format( "failed to get raw id for cached file digest {}", cached_obj.value()->digest.hash())}; } } else { // rehash file auto const file_path = read_path(*file_digest, file_type); if (not file_path) { return unexpected{fmt::format("failed reading CAS entry {}", file_digest->hash())}; } auto const blob_digest = store_file(*file_path, file.is_executable()); if (not blob_digest) { return unexpected{ fmt::format("failed rehashing as blob file {}", file_digest->hash())}; } if (auto raw_id = FromHexString(blob_digest->hash())) { entries[std::move(*raw_id)].emplace_back(file.name(), file_type); } else { return unexpected{fmt::format( "failed to get raw id for file blob digest {}", blob_digest->hash())}; } // cache digest mapping if (auto error_msg = store_rehashed(*file_digest, *blob_digest, file_type)) { return unexpected{*std::move(error_msg)}; } } } // create and store Git tree auto const git_tree = GitRepo::CreateShallowTree(entries); if (not git_tree) { return unexpected{ fmt::format("failed creating Git tree for bazel Directory {}", digest.hash())}; } auto const tree_digest = store_tree(git_tree->second); // cache digest mapping if (auto error_msg = store_rehashed(digest, *tree_digest, ObjectType::Tree)) { return unexpected{*std::move(error_msg)}; } // return digest return *tree_digest; } catch (std::exception const& ex) { return unexpected{fmt::format( "creating Git tree digest unexpectedly failed with:\n{}", ex.what())}; } } auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, TreeStoreFunc const& store_dir, SymlinkStoreFunc const& store_symlink) noexcept -> std::optional { std::vector files{}; std::vector dirs{}; std::vector symlinks{}; auto dir_reader = [&files, &dirs, &symlinks, &root, &store_file, &store_dir, &store_symlink](std::filesystem::path const& name, ObjectType type) { const auto& full_name = root / name; if (IsTreeObject(type)) { // create and store sub directory auto digest = CreateDirectoryDigestFromLocalTree( root / name, store_file, store_dir, store_symlink); if (not digest) { Logger::Log(LogLevel::Error, "failed storing tree {}", full_name.string()); return false; } dirs.emplace_back(CreateDirectoryNode(name.string(), *digest)); return true; } try { if (IsSymlinkObject(type)) { // create and store symlink auto content = FileSystemManager::ReadSymlink(full_name); if (content and store_symlink(*content)) { symlinks.emplace_back( CreateSymlinkNode(name.string(), *content)); return true; } Logger::Log(LogLevel::Error, "failed storing symlink {}", full_name.string()); return false; } // create and store file if (auto digest = store_file(full_name, IsExecutableObject(type))) { auto file = CreateFileNode(name.string(), type, *digest); files.emplace_back(std::move(file)); return true; } Logger::Log( LogLevel::Error, "failed storing file {}", full_name.string()); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "storing file failed with:\n{}", ex.what()); } return false; }; if (FileSystemManager::ReadDirectory( root, dir_reader, /*allow_upwards=*/true)) { auto dir = CreateDirectory(files, dirs, symlinks); if (auto bytes = SerializeMessage(dir)) { try { return store_dir(*bytes); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "storing directory failed with:\n{}", ex.what()); } return std::nullopt; } } return std::nullopt; } auto BazelMsgFactory::CreateGitTreeDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, TreeStoreFunc const& store_tree, SymlinkStoreFunc const& store_symlink) noexcept -> std::optional { GitRepo::tree_entries_t entries{}; auto dir_reader = [&entries, &root, &store_file, &store_tree, &store_symlink](std::filesystem::path const& name, ObjectType type) { const auto full_name = root / name; if (IsTreeObject(type)) { // create and store sub directory if (auto digest = CreateGitTreeDigestFromLocalTree( full_name, store_file, store_tree, store_symlink)) { if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back(name.string(), ObjectType::Tree); return true; } } Logger::Log( LogLevel::Error, "failed storing tree {}", full_name.string()); return false; } try { if (IsSymlinkObject(type)) { auto content = FileSystemManager::ReadSymlink(full_name); if (content and PathIsNonUpwards(*content)) { if (auto digest = store_symlink(*content)) { if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back( name.string(), type); return true; } } Logger::Log(LogLevel::Error, "failed storing symlink {}", full_name.string()); } else { Logger::Log(LogLevel::Error, "failed storing symlink {} -- not non-upwards", full_name.string()); } return false; } // create and store file if (auto digest = store_file(full_name, IsExecutableObject(type))) { if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back(name.string(), type); return true; } } Logger::Log( LogLevel::Error, "failed storing file {}", full_name.string()); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "storing file failed with:\n{}", ex.what()); } return false; }; if (FileSystemManager::ReadDirectory( root, dir_reader, /*allow_upwards=*/true)) { if (auto tree = GitRepo::CreateShallowTree(entries)) { try { return store_tree(tree->second); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "storing tree failed with:\n{}", ex.what()); } return std::nullopt; } } return std::nullopt; } auto BazelMsgFactory::CreateActionDigestFromCommandLine( ActionDigestRequest const& request) -> std::optional { auto cmd = CreateCommandBundle(request); if (not cmd) { return std::nullopt; } auto action = CreateActionBundle(cmd->GetDigest(), request); if (not action) { return std::nullopt; } auto result = action->GetDigest(); if (request.store_blob) { std::invoke(*request.store_blob, *std::move(cmd)); std::invoke(*request.store_blob, *std::move(action)); } return result; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp000066400000000000000000000257501516554100600334160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" /// \brief Factory for creating Bazel API protobuf messages. /// Responsible for creating protobuf messages necessary for Bazel API server /// communication. class BazelMsgFactory { public: /// \brief Store or otherwise process a blob. Returns success flag. using BlobProcessFunc = std::function; using LinkDigestResolveFunc = std::function const&, gsl::not_null*> const&)>; using PathReadFunc = std::function( ArtifactDigest const&, ObjectType)>; using GitReadFunc = std::function>(ArtifactDigest const&, ObjectType)>; using BlobStoreFunc = std::function( std::variant const&, bool)>; using FileStoreFunc = std::function< std::optional(std::filesystem::path const&, bool)>; using SymlinkStoreFunc = std::function(std::string const&)>; using TreeStoreFunc = std::function(std::string const&)>; using RehashedDigestReadFunc = std::function, std::string>(ArtifactDigest const&)>; using RehashedDigestStoreFunc = std::function(ArtifactDigest const&, ArtifactDigest const&, ObjectType)>; /// \brief Create Directory digest from artifact tree structure. Uses /// compatible HashFunction for hashing. Recursively traverse entire tree /// and create blobs for sub-directories. /// \param tree Directory tree of artifacts. /// \param resolve_links Function for resolving symlinks. /// \param process_blob Function for processing Directory blobs. /// \returns Digest representing the entire tree. [[nodiscard]] static auto CreateDirectoryDigestFromTree( DirectoryTreePtr const& tree, LinkDigestResolveFunc const& resolve_links, BlobProcessFunc const& process_blob) noexcept -> std::optional; /// \brief Create Directory digest from an owned Git tree. /// Recursively traverse entire tree and store files and directories. /// Used to convert from native to compatible representation of trees. /// \param digest Digest of a Git tree. /// \param read_git Function for reading Git tree entries. Reading from /// CAS returns the CAS path, while reading from Git CAS /// returns content directly. This differentiation is /// made to avoid unnecessary storing blobs in memory. /// \param store_file Function for storing file via path or content. /// \param store_dir Function for storing Directory blobs. /// \param store_symlink Function for storing symlink via content. /// \param read_rehashed Function to read mapping between digests. /// \param store_rehashed Function to store mapping between digests. /// \returns Digest representing the entire tree directory, or error string /// on failure. [[nodiscard]] static auto CreateDirectoryDigestFromGitTree( ArtifactDigest const& digest, GitReadFunc const& read_git, BlobStoreFunc const& store_file, TreeStoreFunc const& store_dir, SymlinkStoreFunc const& store_symlink, RehashedDigestReadFunc const& read_rehashed, RehashedDigestStoreFunc const& store_rehashed) noexcept -> expected; /// \brief Create Git tree digest from an owned Directory. /// Recursively traverse entire directory and store blobs and trees. /// Used to convert from compatible to native representation of trees. /// \param digest Digest of a bazel directory. /// \param read_path Function for reading CAS path of compatible /// blobs. /// \param store_file Function for storing local file via path. /// \param store_tree Function for storing Git trees. /// \param store_symlink Function for storing symlink via content. /// \param read_rehashed Function for retrieving cached digests. /// \param store_rehashed Function to register digests for caching. /// \returns Digest of a Git tree representing the entire bazel Directory, /// or error string on failure. [[nodiscard]] static auto CreateGitTreeDigestFromDirectory( ArtifactDigest const& digest, PathReadFunc const& read_path, FileStoreFunc const& store_file, TreeStoreFunc const& store_tree, SymlinkStoreFunc const& store_symlink, RehashedDigestReadFunc const& read_rehashed, RehashedDigestStoreFunc const& store_rehashed) noexcept -> expected; /// \brief Create Directory digest from local file root. /// Recursively traverse entire root and store files and directories. /// \param root Path to local file root. /// \param store_file Function for storing local file via path. /// \param store_dir Function for storing Directory blobs. /// \param store_symlink Function for storing symlink via content. /// \returns Digest representing the entire file root. [[nodiscard]] static auto CreateDirectoryDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, TreeStoreFunc const& store_dir, SymlinkStoreFunc const& store_symlink) noexcept -> std::optional; /// \brief Create Git tree digest from local file root. /// Recursively traverse entire root and store files and directories. /// \param root Path to local file root. /// \param store_file Function for storing local file via path. /// \param store_tree Function for storing git trees. /// \param store_symlink Function for storing symlink via content. /// \returns Digest representing the entire file root. [[nodiscard]] static auto CreateGitTreeDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, TreeStoreFunc const& store_tree, SymlinkStoreFunc const& store_symlink) noexcept -> std::optional; struct ActionDigestRequest; /// \brief Creates Action digest from command line. /// As part of the internal process, it creates an ActionBundle and /// CommandBundle that can be captured via BlobStoreFunc. /// \returns Digest representing the action. [[nodiscard]] static auto CreateActionDigestFromCommandLine( ActionDigestRequest const& request) -> std::optional; /// \brief Create message vector from std::map. /// \param[in] input map /// \tparam T protobuf message type. It must be a name-value /// message (i.e. class methods T::set_name(std::string) and /// T::set_value(std::string) must exist) template [[nodiscard]] static auto CreateMessageVectorFromMap( std::map const& input) noexcept -> std::vector { std::vector output{}; std::transform(std::begin(input), std::end(input), std::back_inserter(output), [](auto const& key_val) { T msg; msg.set_name(key_val.first); msg.set_value(key_val.second); return msg; }); return output; } template [[nodiscard]] static auto MessageFromString(std::string const& blob) -> std::optional { T msg{}; if (msg.ParseFromString(blob)) { return msg; } Logger::Log(LogLevel::Error, "failed to parse message from string"); return std::nullopt; } }; struct BazelMsgFactory::ActionDigestRequest final { using BlobStoreFunc = std::function; template using VectorPtr = gsl::not_null const*>; /// \brief The command line. VectorPtr const command_line; /// \brief The workingg direcotry gsl::not_null const cwd; /// \brief The paths of output files. VectorPtr const output_files; /// \brief The paths of output directories. VectorPtr const output_dirs; /// \brief Generic output paths (indicates protocol >=v2.1 should be used). VectorPtr const output_paths; /// \brief The environment variables set. VectorPtr const env_vars; /// \brief The target platform's properties. VectorPtr const properties; /// \brief The Digest of the execution directory. gsl::not_null const exec_dir; /// \brief Hash function to be used. HashFunction const hash_function; /// \brief The command execution timeout. std::chrono::milliseconds const timeout; /// \brief Skip action cache. bool skip_action_cache; /// \brief Function for storing action and cmd bundles. std::optional const store_blob = std::nullopt; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/directory_tree.cpp000066400000000000000000000054211516554100600327330ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "directory_tree.hpp" #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" auto DirectoryTree::AddArtifact(std::filesystem::path const& path, Artifact const* artifact) noexcept -> bool { auto const norm_path = path.lexically_normal(); if (norm_path.empty() or not FileSystemManager::IsRelativePath(norm_path)) { return false; } auto it = norm_path.begin(); try { return AddArtifact(&it, norm_path.end(), artifact); } catch (...) { return false; } } auto DirectoryTree::FromNamedArtifacts( std::vector const& artifacts) noexcept -> std::optional { auto dir_tree = std::make_unique(); for (auto const& [local_path, node] : artifacts) { auto const* artifact = &node->Content(); if (not dir_tree->AddArtifact(local_path, artifact)) { Logger::Log(LogLevel::Error, "failed to add artifact {} ({}) to directory tree", local_path, artifact->Digest().value_or(ArtifactDigest{}).hash()); return std::nullopt; } } return dir_tree; } auto DirectoryTree::AddArtifact(std::filesystem::path::iterator* begin, std::filesystem::path::iterator const& end, Artifact const* artifact) -> bool { ExpectsAudit(std::distance(*begin, end) > 0); auto segment = *((*begin)++); if (segment == "." or segment == "..") { // fail on "." and ".." return false; } if (*begin == end) { return nodes_.emplace(segment, artifact).second; } auto const [it, success] = nodes_.emplace(segment, std::make_unique()); return (success or std::holds_alternative(it->second)) and std::get(it->second) ->AddArtifact(begin, end, artifact); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/directory_tree.hpp000066400000000000000000000047311516554100600327430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" class DirectoryTree; using DirectoryTreePtr = gsl::not_null>; /// \brief Tree of named artifacts. The path through the tree until a leaf node /// where an artifact is stored represents the file path of that artifact. The /// tree can be traversed and converted to, e.g., `BlobTree` or /// `DirectoryNodeBundle`. class DirectoryTree { public: using Node = std::variant; /// \brief Add `Artifact*` to tree. [[nodiscard]] auto AddArtifact(std::filesystem::path const& path, Artifact const* artifact) noexcept -> bool; /// \brief Create a DirectoryTree from a list of named artifacts. [[nodiscard]] static auto FromNamedArtifacts( std::vector const& artifacts) noexcept -> std::optional; [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); } [[nodiscard]] auto end() const noexcept { return nodes_.end(); } [[nodiscard]] auto size() const noexcept { return nodes_.size(); } private: std::unordered_map nodes_; [[nodiscard]] auto AddArtifact(std::filesystem::path::iterator* begin, std::filesystem::path::iterator const& end, Artifact const* artifact) -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/bazel_msg/execution_config.hpp000066400000000000000000000020601516554100600332410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_EXECUTION_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_EXECUTION_CONFIG_HPP /// \file bazel_common.hpp /// \brief Common types and functions required by Bazel API. struct ExecutionConfiguration { int execution_priority{}; int results_cache_priority{}; bool skip_cache_lookup{}; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_EXECUTION_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/000077500000000000000000000000001516554100600265275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/TARGETS000066400000000000000000000110341516554100600275620ustar00rootroot00000000000000{ "common": { "type": ["@", "rules", "CC", "library"] , "name": ["common"] , "hdrs": [ "execution_api.hpp" , "execution_action.hpp" , "execution_response.hpp" , "tree_reader.hpp" , "tree_reader_utils.hpp" , "stream_dumper.hpp" ] , "srcs": ["tree_reader_utils.cpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "tmp_dir"] ] , "private-deps": [ ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "execution_api", "common"] } , "ids": { "type": ["@", "rules", "CC", "library"] , "name": ["common"] , "hdrs": ["ids.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "execution_api", "common"] } , "bytestream_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["bytestream_utils"] , "hdrs": ["bytestream_utils.hpp"] , "srcs": ["bytestream_utils.cpp"] , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/utils/cpp", "expected"] ] , "private-deps": [["@", "fmt", "", "fmt"], ["src/buildtool/common", "bazel_types"]] , "stage": ["src", "buildtool", "execution_api", "common"] } , "api_bundle": { "type": ["@", "rules", "CC", "library"] , "name": ["api_bundle"] , "hdrs": ["api_bundle.hpp"] , "srcs": ["api_bundle.cpp"] , "stage": ["src", "buildtool", "execution_api", "common"] , "deps": [ "common" , ["@", "gsl", "", "gsl"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "config"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/common/remote", "retry_config"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/remote", "context"] ] , "private-deps": [ ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/execution_api/remote", "bazel_api"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/storage", "config"] ] } , "message_limits": { "type": ["@", "rules", "CC", "library"] , "name": ["message_limits"] , "hdrs": ["message_limits.hpp"] , "srcs": ["message_limits.cpp"] , "private-deps": [["@", "grpc", "", "grpc"]] , "stage": ["src", "buildtool", "execution_api", "common"] } , "common_api": { "type": ["@", "rules", "CC", "library"] , "name": ["common_api"] , "hdrs": ["common_api.hpp"] , "srcs": ["common_api.cpp"] , "deps": [ "blob_tree" , "common" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "execution_api", "common"] , "private-deps": [ "message_limits" , ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "back_map"] ] } , "blob_tree": { "type": ["@", "rules", "CC", "library"] , "name": ["blob_tree"] , "hdrs": ["blob_tree.hpp"] , "srcs": ["blob_tree.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] ] , "private-deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "execution_api", "common"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/api_bundle.cpp000066400000000000000000000057531516554100600313470ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/api_bundle.hpp" #include #include #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/storage/config.hpp" /// \note Some logic from MakeRemote is duplicated here as that method cannot /// be used without the hash_function field being set prior to the call. auto ApiBundle::Create( gsl::not_null const& local_context, gsl::not_null const& remote_context, RepositoryConfig const* repo_config) -> ApiBundle { IExecutionApi::Ptr local_api = std::make_shared(local_context, repo_config); IExecutionApi::Ptr remote_api = local_api; if (auto const address = remote_context->exec_config->remote_address) { ExecutionConfiguration config; config.skip_cache_lookup = false; remote_api = std::make_shared( remote_context->exec_config->remote_instance_name, address->host, address->port, remote_context->auth, remote_context->retry_config, config, local_context->storage_config->hash_function, local_api->GetTempSpace()); } return ApiBundle{.local = std::move(local_api), .remote = std::move(remote_api)}; } auto ApiBundle::MakeRemote( std::optional const& address, gsl::not_null const& authentication, gsl::not_null const& retry_config) const -> gsl::not_null { if (address) { ExecutionConfiguration config; config.skip_cache_lookup = false; return std::make_shared("remote-execution", address->host, address->port, authentication, retry_config, config, HashFunction{local->GetHashType()}, local->GetTempSpace()); } return local; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/api_bundle.hpp000066400000000000000000000052371516554100600313510ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_API_BUNDLE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_API_BUNDLE_HPP #include #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/context.hpp" /// \brief Utility structure for instantiation of local and remote apis at the /// same time. If the remote api cannot be instantiated, it falls back to /// exactly the same instance that local api is (&*remote == & *local). struct ApiBundle final { /// \brief Create an ApiBundle instance. /// \note A creator method is used instead of a constructor to allow for /// tests to instantiate ApiBundles with own implementations of the APIs. [[nodiscard]] static auto Create( gsl::not_null const& local_context, gsl::not_null const& remote_context, RepositoryConfig const* repo_config) -> ApiBundle; /// \brief Create a Remote object based on the given arguments. /// \param address The endpoint address. /// \param authentication The remote authentication configuration. /// \param retry_config The retry strategy configuration. /// \returns A configured api: BazelApi if a remote address is given, /// otherwise fall back to the already configured LocalApi instance. [[nodiscard]] auto MakeRemote( std::optional const& address, gsl::not_null const& authentication, gsl::not_null const& retry_config) const -> gsl::not_null; gsl::not_null const local; gsl::not_null const remote; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_API_BUNDLE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/blob_tree.cpp000066400000000000000000000064331516554100600311760ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/blob_tree.hpp" #include #include #include #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" auto BlobTree::FromDirectoryTree(DirectoryTreePtr const& tree, std::filesystem::path const& parent) noexcept -> std::optional { GitRepo::tree_entries_t entries; std::vector nodes; try { entries.reserve(tree->size()); for (auto const& [name, node] : *tree) { if (std::holds_alternative(node)) { auto const& dir = std::get(node); auto blob_tree = FromDirectoryTree(dir, parent / name); if (not blob_tree) { return std::nullopt; } auto raw_id = FromHexString((*blob_tree)->blob_.GetDigest().hash()); if (not raw_id) { return std::nullopt; } entries[std::move(*raw_id)].emplace_back(name, ObjectType::Tree); // Only add tree objects to the blob tree to be uploaded. nodes.emplace_back((*blob_tree)); } else { auto const& artifact = std::get(node); auto const& object_info = artifact->Info(); if (not object_info) { return std::nullopt; } auto raw_id = FromHexString(object_info->digest.hash()); if (not raw_id) { return std::nullopt; } entries[std::move(*raw_id)].emplace_back(name, object_info->type); } } if (auto git_tree = GitRepo::CreateShallowTree(entries)) { auto blob = ArtifactBlob::FromMemory( HashFunction{HashFunction::Type::GitSHA1}, ObjectType::Tree, std::move(git_tree)->second); if (blob.has_value()) { return std::make_shared(*std::move(blob), std::move(nodes)); } } } catch (...) { return std::nullopt; } return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/blob_tree.hpp000066400000000000000000000050711516554100600312000ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" class BlobTree; using BlobTreePtr = gsl::not_null>; /// \brief Tree-like blob container to enable tree-invariant satisfying blob /// upload. class BlobTree { public: explicit BlobTree(ArtifactBlob blob, std::vector nodes) : blob_{std::move(blob)}, nodes_{std::move(nodes)} {} [[nodiscard]] auto Blob() const noexcept -> ArtifactBlob { return blob_; } [[nodiscard]] auto IsTree() const noexcept -> bool { return blob_.GetDigest().IsTree(); } /// \brief Create a `BlobTree` from a `DirectoryTree`. [[nodiscard]] static auto FromDirectoryTree( DirectoryTreePtr const& tree, std::filesystem::path const& parent = "") noexcept -> std::optional; [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); } [[nodiscard]] auto end() const noexcept { return nodes_.end(); } [[nodiscard]] auto size() const noexcept { return nodes_.size(); } [[nodiscard]] auto hash() const { return std::hash{}(blob_); } private: ArtifactBlob blob_; std::vector nodes_; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(BlobTree const& bt) const { return bt.hash(); } }; } // namespace std namespace std { template <> struct hash { [[nodiscard]] auto operator()(BlobTreePtr const& btp) const { return btp->hash(); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/bytestream_utils.cpp000066400000000000000000000171751516554100600326450ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" namespace { /// \brief Split a string into parts with '/' delimiter /// \param request String to be split /// \return A vector of parts on success or an empty vector on failure. [[nodiscard]] auto SplitRequest(std::string const& request) noexcept -> std::vector { std::vector parts; try { std::size_t shift = 0; for (std::size_t length = 0; shift + length < request.size(); ++length) { if (request[shift + length] == '/') { parts.emplace_back(&request[shift], length); shift += length + 1; length = 0; } } if (shift < request.size()) { parts.emplace_back(&request[shift], request.size() - shift); } } catch (...) { return {}; } return parts; } [[nodiscard]] inline auto ToBazelDigest(std::string hash, std::size_t size) noexcept -> bazel_re::Digest { bazel_re::Digest digest{}; digest.set_hash(std::move(hash)); digest.set_size_bytes(static_cast(size)); return digest; } } // namespace const std::set ByteStreamUtils::kOtherReservedFragments = std::set{"actions", "actionResults", "operations", "capabilities", "compressed-blobs"}; auto ByteStreamUtils::ReadRequest::ToString( std::string instance_name, ArtifactDigest const& digest) noexcept -> std::string { if (instance_name.empty()) { return fmt::format("{}/{}/{}", ByteStreamUtils::kBlobs, ArtifactDigestFactory::ToBazel(digest).hash(), digest.size()); } return fmt::format("{}/{}/{}/{}", std::move(instance_name), ByteStreamUtils::kBlobs, ArtifactDigestFactory::ToBazel(digest).hash(), digest.size()); } auto ByteStreamUtils::ReadRequest::FromString( std::string const& request) noexcept -> std::optional { static constexpr std::size_t kHashIndexOffset = 1U; static constexpr std::size_t kSizeIndexOffset = 2U; static constexpr std::size_t kReadRequestPartsCountOffset = 3U; auto const parts = ::SplitRequest(request); int blobs_index = -1; for (int i = 0; i < parts.size(); i++) { if (parts[i].compare(ByteStreamUtils::kBlobs) == 0) { blobs_index = i; break; } if (parts[i].compare(ByteStreamUtils::kUploads) == 0) { // "uploads" not allowed in instance name return std::nullopt; } if (ByteStreamUtils::kOtherReservedFragments.contains( std::string{parts[i]})) { return std::nullopt; } } if (blobs_index < 0) { return std::nullopt; } if (parts.size() != blobs_index + kReadRequestPartsCountOffset) { return std::nullopt; } std::ostringstream instance_name{}; for (int i = 0; i < blobs_index; i++) { instance_name << parts[i]; if (i + 1 < blobs_index) { instance_name << "/"; } } ReadRequest result; result.instance_name_ = instance_name.str(); result.hash_ = std::string(parts[blobs_index + kHashIndexOffset]); try { result.size_ = std::stoul(std::string(parts[blobs_index + kSizeIndexOffset])); } catch (...) { return std::nullopt; } return result; } auto ByteStreamUtils::ReadRequest::GetDigest(HashFunction::Type hash_type) const noexcept -> expected { auto bazel_digest = ToBazelDigest(hash_, size_); return ArtifactDigestFactory::FromBazel(hash_type, bazel_digest); } auto ByteStreamUtils::WriteRequest::ToString( std::string instance_name, std::string uuid, ArtifactDigest const& digest) noexcept -> std::string { if (instance_name.empty()) { return fmt::format("{}/{}/{}/{}/{}", ByteStreamUtils::kUploads, std::move(uuid), ByteStreamUtils::kBlobs, ArtifactDigestFactory::ToBazel(digest).hash(), digest.size()); } return fmt::format("{}/{}/{}/{}/{}/{}", std::move(instance_name), ByteStreamUtils::kUploads, std::move(uuid), ByteStreamUtils::kBlobs, ArtifactDigestFactory::ToBazel(digest).hash(), digest.size()); } auto ByteStreamUtils::WriteRequest::FromString( std::string const& request) noexcept -> std::optional { static constexpr std::size_t kUUIDIndexOffset = 1U; static constexpr std::size_t kBlobsIndexOffset = 2U; static constexpr std::size_t kHashIndexOffset = 3U; static constexpr std::size_t kSizeIndexOffset = 4U; static constexpr std::size_t kWriteRequestPartsCountOffset = 5U; auto const parts = ::SplitRequest(request); int uploads_index = -1; for (int i = 0; i < parts.size(); i++) { if (parts[i].compare(ByteStreamUtils::kUploads) == 0) { uploads_index = i; break; } if (parts[i].compare(ByteStreamUtils::kBlobs) == 0) { // "blobs" not allowed in instance name return std::nullopt; } if (ByteStreamUtils::kOtherReservedFragments.contains( std::string{parts[i]})) { return std::nullopt; } } if (uploads_index < 0) { return std::nullopt; } if ((parts.size() != uploads_index + kWriteRequestPartsCountOffset) or parts[uploads_index + kBlobsIndexOffset].compare( ByteStreamUtils::kBlobs) != 0) { return std::nullopt; } WriteRequest result; std::ostringstream instance_name{}; for (int i = 0; i < uploads_index; i++) { instance_name << parts[i]; if (i + 1 < uploads_index) { instance_name << "/"; } } result.instance_name_ = instance_name.str(); result.uuid_ = std::string(parts[uploads_index + kUUIDIndexOffset]); result.hash_ = std::string(parts[uploads_index + kHashIndexOffset]); try { result.size_ = std::stoul(std::string(parts[uploads_index + kSizeIndexOffset])); } catch (...) { return std::nullopt; } return result; } auto ByteStreamUtils::WriteRequest::GetDigest(HashFunction::Type hash_type) const noexcept -> expected { auto bazel_digest = ToBazelDigest(hash_, size_); return ArtifactDigestFactory::FromBazel(hash_type, bazel_digest); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/bytestream_utils.hpp000066400000000000000000000073601516554100600326450ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/expected.hpp" class ByteStreamUtils final { static constexpr auto* kBlobs = "blobs"; static constexpr auto* kUploads = "uploads"; static const std::set kOtherReservedFragments; public: // Chunk size for uploads (default size used by BuildBarn) static constexpr std::size_t kChunkSize = 64UL * 1024; /// \brief Create a read request for the bytestream service to be /// transferred over the net. Handles serialization/deserialization on its /// own. The pattern is: /// "{instance_name}/{kBlobs}/{digest.hash()}/{digest.size_bytes()}". /// "instance_name_example/blobs/62183d7a696acf7e69e218efc82c93135f8c85f895/4424712" class ReadRequest final { public: [[nodiscard]] static auto ToString( std::string instance_name, ArtifactDigest const& digest) noexcept -> std::string; [[nodiscard]] static auto FromString( std::string const& request) noexcept -> std::optional; [[nodiscard]] auto GetInstanceName() const noexcept -> std::string const& { return instance_name_; } [[nodiscard]] auto GetDigest(HashFunction::Type hash_type) const noexcept -> expected; private: std::string instance_name_; std::string hash_; std::size_t size_ = 0; explicit ReadRequest() = default; }; /// \brief Create a write request for the bytestream service to be /// transferred over the net. Handles serialization/deserialization on its /// own. The pattern is: /// "{instance_name}/{kUploads}/{uuid}/{kBlobs}/{digest.hash()}/{digest.size_bytes()}". /// "instance_name_example/uploads/c4f03510-7d56-4490-8934-01bce1b1288e/blobs/62183d7a696acf7e69e218efc82c93135f8c85f895/4424712" class WriteRequest final { public: [[nodiscard]] static auto ToString( std::string instance_name, std::string uuid, ArtifactDigest const& digest) noexcept -> std::string; [[nodiscard]] static auto FromString( std::string const& request) noexcept -> std::optional; [[nodiscard]] auto GetInstanceName() const noexcept -> std::string const& { return instance_name_; } [[nodiscard]] auto GetUUID() const noexcept -> std::string const& { return uuid_; } [[nodiscard]] auto GetDigest(HashFunction::Type hash_type) const noexcept -> expected; private: std::string instance_name_; std::string uuid_; std::string hash_; std::size_t size_ = 0; explicit WriteRequest() = default; }; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/common_api.cpp000066400000000000000000000232231516554100600313560ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/common_api.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/back_map.hpp" auto CommonRetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, std::function const&)> const& dump_to_stream, std::optional> const& fallback) noexcept -> bool { if (artifacts_info.size() != fds.size()) { Logger::Log(LogLevel::Error, "different number of digests and file descriptors."); return false; } for (std::size_t i{}; i < artifacts_info.size(); ++i) { auto fd = fds[i]; auto const& info = artifacts_info[i]; if (gsl::owner out = fdopen(dup(fd), "wb")) { // NOLINT bool success{false}; try { success = dump_to_stream(info, out); } catch (std::exception const& ex) { std::fclose(out); // close file Logger::Log(LogLevel::Error, "dumping {} to stream failed with:\n{}", info.ToString(), ex.what()); return false; } std::fclose(out); if (not success) { Logger::Log( LogLevel::Debug, "dumping {} {} from CAS to file descriptor {} failed.", IsTreeObject(info.type) ? "tree" : "blob", info.ToString(), fd); // locally we might be able to fallback to Git in native mode try { if (fallback) { success = (*fallback)(info, fd); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "fallback dumping {} to file descriptor {} " "failed with:\n{}", info.ToString(), fd, ex.what()); return false; } } if (not success) { return false; } } else { Logger::Log( LogLevel::Error, "dumping to file descriptor {} failed.", fd); return false; } } return true; } auto CommonUploadBlobTree(BlobTreePtr const& blob_tree, IExecutionApi const& api) noexcept -> bool { // Create digest list from blobs for batch availability check. std::unordered_set missing; missing.reserve(blob_tree->size()); { auto back_map = BackMap::Make( &*blob_tree, [](BlobTreePtr const& node) { return node->Blob().GetDigest(); }); if (back_map == nullptr) { Logger::Log(LogLevel::Error, "Failed to retrieve the missing tree blobs for upload"); return false; } auto missing_digests = api.GetMissingDigests(back_map->GetKeys()); missing = back_map->GetValues(missing_digests); } // Process missing blobs. std::unordered_set container; for (auto const& node : missing) { // Process trees. if (node->IsTree() and not CommonUploadBlobTree(node, api)) { return false; } // Optimize store & upload by taking into account the maximum // transfer size. if (not UpdateContainerAndUpload( &container, node->Blob(), /*exception_is_fatal=*/false, [&api](std::unordered_set&& blobs) -> bool { return api.Upload(std::move(blobs), /*skip_find_missing=*/true); })) { return false; } } // Transfer any remaining blobs. return api.Upload(std::move(container), /*skip_find_missing=*/true); } auto CommonUploadTreeCompatible( IExecutionApi const& api, DirectoryTreePtr const& build_root, BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept -> std::optional { std::unordered_set blobs{}; // Store and upload blobs, taking into account the maximum transfer size. auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( build_root, resolve_links, [&blobs, &api](ArtifactBlob&& blob) { return UpdateContainerAndUpload( &blobs, std::move(blob), /*exception_is_fatal=*/false, [&api](std::unordered_set&& container) -> bool { return api.Upload(std::move(container), /*skip_find_missing=*/false); }); }); if (not digest) { Logger::Log(LogLevel::Debug, "failed to create digest for build root."); return std::nullopt; } Logger::Log(LogLevel::Trace, [&digest]() { std::ostringstream oss{}; oss << "upload root directory" << std::endl; oss << fmt::format(" - root digest: {}", digest->hash()) << std::endl; return oss.str(); }); // Upload remaining blobs. if (not api.Upload(std::move(blobs), /*skip_find_missing=*/false)) { Logger::Log(LogLevel::Debug, "failed to upload blobs for build root."); return std::nullopt; } return digest; } auto CommonUploadTreeNative(IExecutionApi const& api, DirectoryTreePtr const& build_root) noexcept -> std::optional { auto blob_tree = BlobTree::FromDirectoryTree(build_root); if (not blob_tree) { Logger::Log(LogLevel::Debug, "failed to create blob tree for build root."); return std::nullopt; } auto tree_blob = (*blob_tree)->Blob(); // Upload blob tree if tree is not available at the remote side (content // first). if (not api.IsAvailable(tree_blob.GetDigest())) { if (not CommonUploadBlobTree(*blob_tree, api)) { Logger::Log(LogLevel::Debug, "failed to upload blob tree for build root."); return std::nullopt; } if (not api.Upload({tree_blob}, /*skip_find_missing=*/true)) { Logger::Log(LogLevel::Debug, "failed to upload tree blob for build root."); return std::nullopt; } } return tree_blob.GetDigest(); } auto UpdateContainerAndUpload( gsl::not_null*> const& container, ArtifactBlob&& blob, bool exception_is_fatal, std::function&&)> const& uploader, Logger const* logger) noexcept -> bool { // Optimize upload of blobs with respect to the maximum transfer limit, such // that we never store unnecessarily more data in the container than we need // per remote transfer. try { if (blob.GetContentSize() > MessageLimits::kMaxGrpcLength) { // large blobs use individual stream upload if (not uploader( std::unordered_set{{std::move(blob)}})) { return false; } } else { if (not container->contains(blob)) { std::size_t content_size = 0; for (auto const& blob : *container) { content_size += blob.GetContentSize(); } if (content_size + blob.GetContentSize() > MessageLimits::kMaxGrpcLength) { // swap away from original container to allow move during // upload std::unordered_set tmp_container{}; std::swap(*container, tmp_container); // if we would surpass the transfer limit, upload the // current container and clear it before adding more blobs if (not uploader(std::move(tmp_container))) { return false; } } // add current blob to container container->emplace(std::move(blob)); } } } catch (std::exception const& ex) { if (exception_is_fatal) { Logger::Log(logger, LogLevel::Error, "failed to emplace blob with\n:{}", ex.what()); } return false; } return true; // success! } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/common_api.hpp000066400000000000000000000077001516554100600313650ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/execution_api/common/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/logging/logger.hpp" /// \brief Common logic for RetrieveToFds. /// \param dump_to_stream Dumps the artifact to the respective open stream. /// \param fallback Processes the respective file descriptor further in case the /// regular dump fails. [[nodiscard]] auto CommonRetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, std::function const&)> const& dump_to_stream, std::optional> const& fallback) noexcept -> bool; /// \brief Upload missing blobs from a given BlobTree. [[nodiscard]] auto CommonUploadBlobTree(BlobTreePtr const& blob_tree, IExecutionApi const& api) noexcept -> bool; /// \brief Runs the compatible branch of local/bazel UploadTree API. [[nodiscard]] auto CommonUploadTreeCompatible( IExecutionApi const& api, DirectoryTreePtr const& build_root, BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept -> std::optional; /// \brief Runs the native branch of local/bazel UploadTree API. [[nodiscard]] auto CommonUploadTreeNative( IExecutionApi const& api, DirectoryTreePtr const& build_root) noexcept -> std::optional; /// \brief Updates the given container based on the given blob, ensuring the /// container is kept under the maximum transfer limit. If the given blob is /// larger than the transfer limit, it is immediately uploaded. Otherwise, /// it is added to the container if it fits inside the transfer limit, or it /// is added to a new container moving forward, with the old one being uploaded. /// This way we ensure we only store as much data as we can actually transfer in /// one go. /// \param container Stores blobs smaller than the transfer limit. /// \param blob New blob to be handled (uploaded or added to container). /// \param exception_is_fatal If true, caught exceptions are logged to Error. /// \param uploader Lambda handling the actual upload call. /// \param logger Use this instance for any logging. If nullptr, use the default /// logger. This value is used only if exception_is_fatal==true. /// \returns Returns true on success, false otherwise (failures or exceptions). [[nodiscard]] auto UpdateContainerAndUpload( gsl::not_null*> const& container, ArtifactBlob&& blob, bool exception_is_fatal, std::function&&)> const& uploader, Logger const* logger = nullptr) noexcept -> bool; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/execution_action.hpp000066400000000000000000000054571516554100600326130ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP #include #include #include #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/logging/logger.hpp" /// \brief Abstract action. /// Can execute multiple commands. Commands are executed in arbitrary order and /// cannot depend on each other. class IExecutionAction { public: using Ptr = std::unique_ptr; enum class CacheFlag : std::uint8_t { CacheOutput, ///< run and cache, or serve from cache DoNotCacheOutput, ///< run and do not cache, never served from cached FromCacheOnly, ///< do not run, only serve from cache PretendCached ///< always run, respond same action id as if cached }; static constexpr std::chrono::milliseconds kDefaultTimeout{1000}; [[nodiscard]] static constexpr auto CacheEnabled(CacheFlag f) -> bool { return f == CacheFlag::CacheOutput or f == CacheFlag::FromCacheOnly; } [[nodiscard]] static constexpr auto ExecutionEnabled(CacheFlag f) -> bool { return f == CacheFlag::CacheOutput or f == CacheFlag::DoNotCacheOutput or f == CacheFlag::PretendCached; } IExecutionAction() = default; IExecutionAction(IExecutionAction const&) = delete; IExecutionAction(IExecutionAction&&) = delete; auto operator=(IExecutionAction const&) -> IExecutionAction& = delete; auto operator=(IExecutionAction&&) -> IExecutionAction& = delete; virtual ~IExecutionAction() = default; /// \brief Execute the action. /// \returns Execution response, with commands' outputs and artifacts. /// \returns nullptr if execution failed. // NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] virtual auto Execute(Logger const* logger = nullptr) noexcept -> IExecutionResponse::Ptr = 0; virtual void SetCacheFlag(CacheFlag flag) noexcept = 0; virtual void SetTimeout(std::chrono::milliseconds timeout) noexcept = 0; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/execution_api.hpp000066400000000000000000000164671516554100600321120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP #include #include #include #include #include #include #include #include #include "src/buildtool/common/artifact.hpp" // Artifact::ObjectInfo #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief Abstract remote execution API /// Can be used to create actions. class IExecutionApi { public: using Ptr = std::shared_ptr; IExecutionApi() = default; IExecutionApi(IExecutionApi const&) = delete; IExecutionApi(IExecutionApi&&) = default; auto operator=(IExecutionApi const&) -> IExecutionApi& = delete; auto operator=(IExecutionApi&&) -> IExecutionApi& = default; virtual ~IExecutionApi() = default; /// \brief Create a new action (>=RBEv2.0). /// \param[in] root_digest Digest of the build root. /// \param[in] command Command as argv vector /// \param[in] cwd Working directory, relative to execution root /// \param[in] output_files List of paths to output files, relative to cwd /// \param[in] output_dirs List of paths to output directories. /// \param[in] env_vars The environment variables to set. /// \param[in] properties Platform properties to set. /// \param[in] force_legacy Force use of legacy API RBEv2.0 /// \returns The new action. /// Note that types of output files and directories are not verified. /// NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] virtual auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, std::map const& properties, bool force_legacy = false) const noexcept -> IExecutionAction::Ptr = 0; /// \brief Retrieve artifacts from CAS and store to specified paths. /// Tree artifacts are resolved its containing file artifacts are /// recursively retrieved. /// If the alternative is provided, it can be assumed that this /// alternative CAS is more close, but it might not contain all the /// needed artifacts. /// NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] virtual auto RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* alternative = nullptr) const noexcept -> bool = 0; /// \brief Retrieve artifacts from CAS and write to file descriptors. /// Tree artifacts are not resolved and instead the tree object will be /// pretty-printed before writing to fd. If `raw_tree` is set, pretty /// printing will be omitted and the raw tree object will be written /// instead. /// NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] virtual auto RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree, IExecutionApi const* alternative = nullptr) const noexcept -> bool = 0; /// \brief Synchronization of artifacts between two CASes. Retrieves /// artifacts from one CAS and writes to another CAS. Tree artifacts are /// resolved and its containing file artifacts are recursively retrieved. [[nodiscard]] virtual auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool = 0; /// \brief A variant of RetrieveToCas that is allowed to internally use /// the specified number of threads to carry out the task in parallel. /// Given it is supported by the server, blob splitting enables traffic /// reduction when fetching blobs from the remote by reusing locally /// available blob chunks and just fetching unknown blob chunks to assemble /// the remote blobs. [[nodiscard]] virtual auto ParallelRetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api, std::size_t /* jobs */, bool /* use_blob_splitting */) const noexcept -> bool { return RetrieveToCas(artifacts_info, api); } /// \brief Retrieve one artifact from CAS and make it available for /// furter in-memory processing [[nodiscard]] virtual auto RetrieveToMemory( Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional = 0; /// \brief Upload blobs to CAS. Uploads only the blobs that are not yet /// available in CAS, unless `skip_find_missing` is specified. /// \param blobs Container of blobs to upload. /// \param skip_find_missing Skip finding missing blobs, just upload all. /// NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] virtual auto Upload( std::unordered_set&& blobs, bool skip_find_missing = false) const noexcept -> bool = 0; [[nodiscard]] virtual auto UploadTree( std::vector const& artifacts) const noexcept -> std::optional = 0; [[nodiscard]] virtual auto IsAvailable( ArtifactDigest const& digest) const noexcept -> bool = 0; [[nodiscard]] virtual auto GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set = 0; [[nodiscard]] virtual auto SplitBlob(ArtifactDigest const& /*blob_digest*/) const noexcept -> std::optional> { return std::nullopt; } [[nodiscard]] virtual auto BlobSplitSupport() const noexcept -> bool { return false; } [[nodiscard]] virtual auto SpliceBlob( ArtifactDigest const& /*blob_digest*/, std::vector const& /*chunk_digests*/) const noexcept -> std::optional { return std::nullopt; } [[nodiscard]] virtual auto BlobSpliceSupport() const noexcept -> bool { return false; } [[nodiscard]] virtual auto GetHashType() const noexcept -> HashFunction::Type = 0; [[nodiscard]] virtual auto GetTempSpace() const noexcept -> TmpDir::Ptr = 0; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/execution_response.hpp000066400000000000000000000061521516554100600331650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/utils/cpp/expected.hpp" /// \brief Abstract response. /// Response of an action execution. Contains outputs from multiple commands and /// a single container with artifacts. class IExecutionResponse { public: using Ptr = std::unique_ptr; using ArtifactInfos = std::unordered_map; // set of paths found in output_directory_symlinks list of the action result enum class StatusCode : std::uint8_t { Failed, Success }; IExecutionResponse() = default; IExecutionResponse(IExecutionResponse const&) = delete; IExecutionResponse(IExecutionResponse&&) = delete; auto operator=(IExecutionResponse const&) -> IExecutionResponse& = delete; auto operator=(IExecutionResponse&&) -> IExecutionResponse& = delete; virtual ~IExecutionResponse() = default; [[nodiscard]] virtual auto Status() const noexcept -> StatusCode = 0; [[nodiscard]] virtual auto ExitCode() const noexcept -> int = 0; [[nodiscard]] virtual auto IsCached() const noexcept -> bool = 0; [[nodiscard]] virtual auto HasStdErr() const noexcept -> bool = 0; [[nodiscard]] virtual auto HasStdOut() const noexcept -> bool = 0; [[nodiscard]] virtual auto StdErrDigest() noexcept -> std::optional = 0; [[nodiscard]] virtual auto StdOutDigest() noexcept -> std::optional = 0; [[nodiscard]] virtual auto StdErr() noexcept -> std::string = 0; [[nodiscard]] virtual auto StdOut() noexcept -> std::string = 0; // Duration of the actual action execution, in seconds. The value may // be 0 if the action was taken from cache. [[nodiscard]] virtual auto ExecutionDuration() noexcept -> double = 0; [[nodiscard]] virtual auto ActionDigest() const noexcept -> std::string const& = 0; [[nodiscard]] virtual auto Artifacts() noexcept -> expected, std::string> = 0; [[nodiscard]] virtual auto HasUpwardsSymlinks() noexcept -> expected = 0; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/ids.hpp000066400000000000000000000111351516554100600300200ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_IDS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_IDS_HPP #ifdef __unix__ #include #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/hex_string.hpp" /// \brief Create unique ID for current process and thread. [[nodiscard]] static inline auto CreateProcessUniqueId() noexcept -> std::optional { #ifdef __unix__ pid_t pid{}; try { pid = getpid(); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return std::nullopt; } #endif auto tid = std::this_thread::get_id(); std::ostringstream id{}; id << pid << "-" << tid; return id.str(); } /// \brief Create unique path based on file_path. [[nodiscard]] static inline auto CreateUniquePath( std::filesystem::path file_path) noexcept -> std::optional { auto id = CreateProcessUniqueId(); if (id) { return file_path.concat("." + *id); } return std::nullopt; } [[nodiscard]] static auto GetNonDeterministicRandomNumber() -> unsigned int { std::uniform_int_distribution dist{}; std::random_device urandom{ #ifdef __unix__ "/dev/urandom" #endif }; return dist(urandom); } static auto const kRandomConstant = GetNonDeterministicRandomNumber(); static void EncodeUUIDVersion4(std::string* uuid) { constexpr auto kVersionByte = 6UL; constexpr auto kVersionBits = 0x40U; // version 4: 0100 xxxx constexpr auto kClearMask = 0x0fU; Expects(uuid->size() >= kVersionByte); auto& byte = uuid->at(kVersionByte); byte = static_cast(kVersionBits | (kClearMask & static_cast(byte))); } static void EncodeUUIDVariant1(std::string* uuid) { constexpr auto kVariantByte = 8UL; constexpr auto kVariantBits = 0x80U; // variant 1: 10xx xxxx constexpr auto kClearMask = 0x3fU; Expects(uuid->size() >= kVariantByte); auto& byte = uuid->at(kVariantByte); byte = static_cast(kVariantBits | (kClearMask & static_cast(byte))); } /// \brief Create UUID version 4 from seed. [[nodiscard]] static inline auto CreateUUIDVersion4(std::string const& seed) -> std::string { constexpr auto kRawLength = 16UL; constexpr auto kHexDashPos = std::array{8UL, 12UL, 16UL, 20UL}; // The type of HashFunction is irrelevant here. It is used for // identification purposes only. SHA256 is used. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; auto value = fmt::format("{}-{}", std::to_string(kRandomConstant), seed); auto uuid = hash_function.PlainHashData(value).Bytes(); EncodeUUIDVersion4(&uuid); EncodeUUIDVariant1(&uuid); Expects(uuid.size() >= kRawLength); std::size_t cur{}; std::ostringstream ss{}; auto uuid_hex = ToHexString(uuid.substr(0, kRawLength)); for (auto pos : kHexDashPos) { ss << uuid_hex.substr(cur, pos - cur) << '-'; cur = pos; } ss << uuid_hex.substr(cur); EnsuresAudit(ss.str().size() == (2 * kRawLength) + kHexDashPos.size()); return ss.str(); } /// \brief Create a UUID for the current process [[nodiscard]] static inline auto CreateUUID() -> std::string { auto process_seed = CreateProcessUniqueId(); // as CreateUUIDVersion4 still uses the process-specific random number, // we can afford to use a constant seed. return CreateUUIDVersion4(process_seed ? *process_seed : "unknown"); } #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_IDS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/message_limits.cpp000066400000000000000000000015421516554100600322420ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/message_limits.hpp" #include static_assert(MessageLimits::kMaxGrpcLength < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH, "Max batch transfer size too large."); just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/message_limits.hpp000066400000000000000000000017511516554100600322510ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_MESSAGE_LIMITS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_MESSAGE_LIMITS_HPP #include struct MessageLimits final { // Maximum length of a gprc message. static constexpr std::size_t kMaxGrpcLength = 3UL * 1024 * 1024; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_MESSAGE_LIMITS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/stream_dumper.hpp000066400000000000000000000074621516554100600321200ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_STREAM_DUMPER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_STREAM_DUMPER_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" template class StreamDumper final { public: template explicit StreamDumper(Args&&... args) noexcept : impl_(std::forward(args)...) {} /// \brief Dump artifact to file stream. /// Tree artifacts are pretty-printed (i.e., contents are listed) unless /// raw_tree is set, then the raw tree will be written to the file stream. /// \param info The object info of the artifact to dump. /// \param stream The file stream to dump to. /// \param raw_tree Dump tree as raw blob. /// \returns true on success. [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, gsl::not_null const& stream, bool raw_tree) const noexcept -> bool { const bool is_tree = IsTreeObject(info.type); if (is_tree and raw_tree) { return DumpRawTree(info, stream); } return is_tree ? DumpTree(info, stream) : DumpBlob(info, stream); } private: TImpl impl_; [[nodiscard]] auto DumpRawTree( Artifact::ObjectInfo const& info, gsl::not_null const& stream) const noexcept -> bool { auto writer = [this, &stream](std::string const& data) -> bool { return DumpString(data, stream); }; return impl_.DumpRawTree(info, writer); } [[nodiscard]] auto DumpTree( Artifact::ObjectInfo const& info, gsl::not_null const& stream) const noexcept -> bool { if (not impl_.IsNativeProtocol()) { auto directory = impl_.ReadDirectory(info.digest); auto data = directory ? TreeReaderUtils::DirectoryToString(*directory) : std::nullopt; if (data) { return DumpString(*data, stream); } } else { auto entries = impl_.ReadGitTree(info.digest); auto data = entries ? TreeReaderUtils::GitTreeToString(*entries) : std::nullopt; if (data) { return DumpString(*data, stream); } } return false; } [[nodiscard]] auto DumpBlob( Artifact::ObjectInfo const& info, gsl::not_null const& stream) const noexcept -> bool { auto writer = [this, &stream](std::string const& data) -> bool { return DumpString(data, stream); }; return impl_.DumpBlob(info, writer); } [[nodiscard]] auto DumpString( std::string const& data, gsl::not_null const& stream) const noexcept -> bool { return std::fwrite(data.data(), 1, data.size(), stream) == data.size(); } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_STREAM_DUMPER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/tree_reader.hpp000066400000000000000000000156061516554100600315310ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP #include #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" struct ReadTreeResult final { std::vector paths; std::vector infos; }; template class TreeReader final { public: template explicit TreeReader(Args&&... args) noexcept : impl_(std::forward(args)...) {} /// \brief Reads the flat content of a tree and returns object infos of all /// its direct entries (trees and blobs). /// \param tree_digest Digest of the tree. /// \param parent Local parent path. /// \returns A struct containing filesystem paths and object infos. [[nodiscard]] auto ReadDirectTreeEntries( ArtifactDigest const& digest, std::filesystem::path const& parent) const noexcept -> std::optional { ReadTreeResult result; TreeReaderUtils::InfoStoreFunc store_info = [&result, &parent](std::filesystem::path const& path, Artifact::ObjectInfo&& info) { result.paths.emplace_back(parent / path); result.infos.emplace_back(std::move(info)); return true; }; if (not impl_.IsNativeProtocol()) { auto tree = impl_.ReadDirectory(digest); if (tree and not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } else { auto tree = impl_.ReadGitTree(digest); if (tree and not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } return result; } /// \brief Traverses a tree recursively and retrieves object infos of all /// found blobs (leafs). Tree objects are by default not added to the result /// list, but converted to a path name. /// \param tree_digest Digest of the tree. /// \param parent Local parent path. /// \param include_trees Include leaf tree objects (empty trees). /// \returns A struct containing filesystem paths and object infos. [[nodiscard]] auto RecursivelyReadTreeLeafs( ArtifactDigest const& digest, std::filesystem::path const& parent, bool include_trees = false) const noexcept -> std::optional { ReadTreeResult result; auto store = [&result](std::filesystem::path const& path, Artifact::ObjectInfo const& info) { result.paths.emplace_back(path); result.infos.emplace_back(info); return true; }; try { if (ReadObjectInfosRecursively( store, parent, digest, include_trees)) { return result; } return std::nullopt; } catch (...) { return std::nullopt; } } /// \brief Traverse a tree recursively and stage all artifacts to paths. /// \param infos Infos to be staged /// \param outputs Paths to be used for staging /// \return True if outputs contain corresponding infos. [[nodiscard]] auto StageTo(std::vector const& infos, std::vector const& outputs) const noexcept -> bool { if (infos.size() != outputs.size()) { return false; } for (std::size_t i = 0; i < infos.size(); ++i) { auto const& info = infos[i]; if (IsTreeObject(info.type)) { auto result = RecursivelyReadTreeLeafs(info.digest, outputs[i]); if (not result or not StageTo(result->infos, result->paths)) { return false; } } else if (not impl_.StageBlobTo(info, outputs[i])) { return false; } } return true; } private: TImpl impl_; [[nodiscard]] static auto IsDirectoryEmpty( bazel_re::Directory const& dir) noexcept -> bool { return dir.files().empty() and dir.directories().empty() and dir.symlinks().empty(); } [[nodiscard]] auto ReadObjectInfosRecursively( TreeReaderUtils::InfoStoreFunc const& store, std::filesystem::path const& parent, ArtifactDigest const& digest, bool const include_trees) const -> bool { TreeReaderUtils::InfoStoreFunc internal_store = [this, &store, &parent, include_trees]( std::filesystem::path const& path, Artifact::ObjectInfo&& info) -> bool { return IsTreeObject(info.type) ? ReadObjectInfosRecursively( store, parent / path, info.digest, include_trees) : store(parent / path, std::move(info)); }; if (not impl_.IsNativeProtocol()) { if (auto tree = impl_.ReadDirectory(digest)) { if (include_trees and IsDirectoryEmpty(*tree)) { if (not store(parent, {digest, ObjectType::Tree})) { return false; } } return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } else { if (auto tree = impl_.ReadGitTree(digest)) { if (include_trees and tree->empty()) { if (not store(parent, {digest, ObjectType::Tree})) { return false; } } return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } return false; } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/tree_reader_utils.cpp000066400000000000000000000150121516554100600327330ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { [[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, bazel_re::DirectoryNode const& node) -> std::optional { auto digest = ArtifactDigestFactory::FromBazel(hash_function.GetType(), node.digest()); if (not digest) { return std::nullopt; } return Artifact::ObjectInfo{.digest = *std::move(digest), .type = ObjectType::Tree}; } [[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, bazel_re::FileNode const& node) -> std::optional { auto digest = ArtifactDigestFactory::FromBazel(hash_function.GetType(), node.digest()); if (not digest) { return std::nullopt; } return Artifact::ObjectInfo{.digest = *std::move(digest), .type = node.is_executable() ? ObjectType::Executable : ObjectType::File}; } [[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, bazel_re::SymlinkNode const& node) -> Artifact::ObjectInfo { return Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs( hash_function, node.target()), .type = ObjectType::Symlink}; } template [[nodiscard]] auto TreeToString(TTree const& entries) -> std::optional { auto json = nlohmann::json::object(); TreeReaderUtils::InfoStoreFunc store_infos = [&json](std::filesystem::path const& path, Artifact::ObjectInfo&& info) -> bool { static constexpr bool kSizeUnknown = std::is_same_v; json[path.string()] = std::move(info).ToString(kSizeUnknown); return true; }; if (TreeReaderUtils::ReadObjectInfos(entries, store_infos)) { return json.dump(2) + "\n"; } Logger::Log(LogLevel::Error, "reading object infos from Directory failed"); return std::nullopt; } } // namespace auto TreeReaderUtils::ReadObjectInfos(bazel_re::Directory const& dir, InfoStoreFunc const& store_info) noexcept -> bool { // SHA256 is used since bazel types are processed here. HashFunction const hash_function{HashFunction::Type::PlainSHA256}; try { for (auto const& f : dir.files()) { auto info = CreateObjectInfo(hash_function, f); if (not info or not store_info(f.name(), *std::move(info))) { return false; } } for (auto const& l : dir.symlinks()) { if (not store_info(l.name(), CreateObjectInfo(hash_function, l))) { return false; } } for (auto const& d : dir.directories()) { auto info = CreateObjectInfo(hash_function, d); if (not info or not store_info(d.name(), *std::move(info))) { return false; } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading object infos from Directory failed with:\n{}", ex.what()); return false; } return true; } auto TreeReaderUtils::ReadObjectInfos(GitRepo::tree_entries_t const& entries, InfoStoreFunc const& store_info) noexcept -> bool { try { for (auto const& [raw_id, es] : entries) { auto const hex_id = ToHexString(raw_id); for (auto const& entry : es) { auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, hex_id, /*size is unknown*/ 0, IsTreeObject(entry.type)); if (not digest or not store_info( entry.name, Artifact::ObjectInfo{.digest = *std::move(digest), .type = entry.type})) { return false; } } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading object infos from Git tree failed with:\n{}", ex.what()); return false; } return true; } auto TreeReaderUtils::DirectoryToString(bazel_re::Directory const& dir) noexcept -> std::optional { try { return TreeToString(dir); } catch (const std::exception& e) { Logger::Log(LogLevel::Error, "An error occurred while reading bazel:re::Directory:\n", e.what()); return std::nullopt; } } auto TreeReaderUtils::GitTreeToString( GitRepo::tree_entries_t const& entries) noexcept -> std::optional { try { return TreeToString(entries); } catch (const std::exception& e) { Logger::Log(LogLevel::Error, "An error occurred while reading git tree:\n{}", e.what()); return std::nullopt; } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/common/tree_reader_utils.hpp000066400000000000000000000041371516554100600327460ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP #include #include #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/git_repo.hpp" class TreeReaderUtils final { public: using InfoStoreFunc = std::function; /// \brief Read object infos from directory. /// \returns true on success. [[nodiscard]] static auto ReadObjectInfos( bazel_re::Directory const& dir, InfoStoreFunc const& store_info) noexcept -> bool; /// \brief Read object infos from git tree. /// \returns true on success. [[nodiscard]] static auto ReadObjectInfos( GitRepo::tree_entries_t const& entries, InfoStoreFunc const& store_info) noexcept -> bool; /// \brief Create descriptive string from Directory protobuf message. [[nodiscard]] static auto DirectoryToString( bazel_re::Directory const& dir) noexcept -> std::optional; /// \brief Create descriptive string from Git tree entries. [[nodiscard]] static auto GitTreeToString( GitRepo::tree_entries_t const& entries) noexcept -> std::optional; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/000077500000000000000000000000001516554100600307625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/TARGETS000066400000000000000000000175551516554100600320330ustar00rootroot00000000000000{ "execution_server": { "type": ["@", "rules", "CC", "library"] , "name": ["execution_server"] , "hdrs": ["execution_server.hpp"] , "srcs": ["execution_server.cpp"] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "googleapis", "", "google_longrunning_operations_proto"] , ["@", "googleapis", "", "google_rpc_status_proto"] ] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ "operation_cache" , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "garbage_collector"] , ["src/utils/cpp", "hex_string"] ] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } , "ac_server": { "type": ["@", "rules", "CC", "library"] , "name": ["ac_server"] , "hdrs": ["ac_server.hpp"] , "srcs": ["ac_server.cpp"] , "proto": [["@", "bazel_remote_apis", "", "remote_execution_proto"]] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "garbage_collector"] , ["src/utils/cpp", "expected"] ] } , "cas_server": { "type": ["@", "rules", "CC", "library"] , "name": ["cas_server"] , "hdrs": ["cas_server.hpp"] , "srcs": ["cas_server.cpp"] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "googleapis", "", "google_rpc_status_proto"] ] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "private-deps": [ "cas_utils" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "garbage_collector"] , ["src/utils/cpp", "expected"] ] } , "server_implementation": { "type": ["@", "rules", "CC", "library"] , "name": ["server_implementation"] , "hdrs": ["server_implementation.hpp"] , "srcs": ["server_implementation.cpp"] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/remote", "context"] ] , "private-deps": [ "ac_server" , "bytestream_server" , "capabilities_server" , "cas_server" , "execution_server" , "operations_server" , ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common/remote", "port"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/file_system", "atomic"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "type_safe_arithmetic"] ] } , "bytestream_server": { "type": ["@", "rules", "CC", "library"] , "name": ["bytestream"] , "hdrs": ["bytestream_server.hpp"] , "srcs": ["bytestream_server.cpp"] , "proto": [["@", "googleapis", "", "google_bytestream_proto"]] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "private-deps": [ "cas_utils" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "bytestream_utils"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "garbage_collector"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] , ["src/utils/cpp", "tmp_dir"] ] } , "capabilities_server": { "type": ["@", "rules", "CC", "library"] , "name": ["capabilities_server"] , "hdrs": ["capabilities_server.hpp"] , "srcs": ["capabilities_server.cpp"] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "bazel_remote_apis", "", "semver_proto"] ] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/crypto", "hash_function"] ] , "private-deps": [ ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/common", "message_limits"] ] } , "operation_cache": { "type": ["@", "rules", "CC", "library"] , "name": ["operation_cache"] , "hdrs": ["operation_cache.hpp"] , "srcs": ["operation_cache.cpp"] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "proto": [["@", "googleapis", "", "google_longrunning_operations_proto"]] , "deps": [["@", "protoc", "", "libprotobuf"]] } , "operations_server": { "type": ["@", "rules", "CC", "library"] , "name": ["operations_server"] , "hdrs": ["operations_server.hpp"] , "srcs": ["operations_server.cpp"] , "deps": [ "operation_cache" , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/logging", "logging"] ] , "proto": [["@", "googleapis", "", "google_longrunning_operations_proto"]] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "hex_string"] ] } , "cas_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["cas_utils"] , "hdrs": ["cas_utils.hpp"] , "srcs": ["cas_utils.cpp"] , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["src/buildtool/common", "common"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/ac_server.cpp000066400000000000000000000056001516554100600334400ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/ac_server.hpp" #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/expected.hpp" auto ActionCacheServiceImpl::GetActionResult( ::grpc::ServerContext* /*context*/, const ::bazel_re::GetActionResultRequest* request, ::bazel_re::ActionResult* response) -> ::grpc::Status { auto action_digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), request->action_digest()); if (not action_digest) { logger_.Emit(LogLevel::Debug, "{}", action_digest.error()); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, std::move(action_digest).error()}; } logger_.Emit(LogLevel::Debug, [&action_digest, request]() { return fmt::format( "GetActionResult(instance_name={}, action_digest={})", nlohmann::json(request->instance_name()).dump(), action_digest->hash()); }); auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "Could not acquire SharedLock"; logger_.Emit(LogLevel::Error, kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } auto action_result = storage_.ActionCache().CachedResult(*action_digest); if (not action_result) { return grpc::Status{ grpc::StatusCode::NOT_FOUND, fmt::format("{} missing from AC", action_digest->hash())}; } *response = *std::move(action_result); return ::grpc::Status::OK; } auto ActionCacheServiceImpl::UpdateActionResult( ::grpc::ServerContext* /*context*/, const ::bazel_re::UpdateActionResultRequest* /*request*/, ::bazel_re::ActionResult* /*response*/) -> ::grpc::Status { static auto constexpr kStr = "UpdateActionResult not implemented"; logger_.Emit(LogLevel::Error, kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/ac_server.hpp000066400000000000000000000062651516554100600334550ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef AC_SERVER_HPP #define AC_SERVER_HPP #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" class ActionCacheServiceImpl final : public bazel_re::ActionCache::Service { public: explicit ActionCacheServiceImpl( gsl::not_null const& local_context) noexcept : storage_config_{*local_context->storage_config}, storage_{*local_context->storage} {} // Retrieve a cached execution result. // // Implementations SHOULD ensure that any blobs referenced from the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] // are available at the time of returning the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] and will be // for some period of time afterwards. The TTLs of the referenced blobs // SHOULD be increased if necessary and applicable. // // Errors: // // * `NOT_FOUND`: The requested `ActionResult` is not in the cache. auto GetActionResult(::grpc::ServerContext* context, const ::bazel_re::GetActionResultRequest* request, ::bazel_re::ActionResult* response) -> ::grpc::Status override; // Upload a new execution result. // // In order to allow the server to perform access control based on the type // of action, and to assist with client debugging, the client MUST first // upload the [Action][build.bazel.remote.execution.v2.Execution] that // produced the result, along with its // [Command][build.bazel.remote.execution.v2.Command], into the // `ContentAddressableStorage`. // // Errors: // // * `INVALID_ARGUMENT`: One or more arguments are invalid. // * `FAILED_PRECONDITION`: One or more errors occurred in updating the // action result, such as a missing command or action. // * `RESOURCE_EXHAUSTED`: There is insufficient storage space to add the // entry to the cache. auto UpdateActionResult( ::grpc::ServerContext* context, const ::bazel_re::UpdateActionResultRequest* request, ::bazel_re::ActionResult* response) -> ::grpc::Status override; private: StorageConfig const& storage_config_; Storage const& storage_; Logger logger_{"execution-service"}; }; #endif // AC_SERVER_HPP bytestream_server.cpp000066400000000000000000000176011516554100600351610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/bytestream_server.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "google/protobuf/stubs/port.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/incremental_reader.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto BytestreamServiceImpl::Read( ::grpc::ServerContext* /*context*/, const ::google::bytestream::ReadRequest* request, ::grpc::ServerWriter<::google::bytestream::ReadResponse>* writer) -> ::grpc::Status { logger_.Emit( LogLevel::Debug, "Read(resource_name={})", request->resource_name()); auto const read_request = ByteStreamUtils::ReadRequest::FromString(request->resource_name()); if (not read_request) { auto const str = fmt::format("could not parse {}", request->resource_name()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } auto const read_digest = read_request->GetDigest(storage_config_.hash_function.GetType()); if (not read_digest) { logger_.Emit(LogLevel::Debug, "{}", read_digest.error()); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, read_digest.error()}; } logger_.Emit(LogLevel::Debug, [&read_request, &read_digest]() { return fmt::format( "Read(instance_name={}, digest={})", nlohmann::json(read_request->GetInstanceName()).dump(), read_digest->hash()); }); auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "Could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } auto const path = read_digest->IsTree() ? storage_.CAS().TreePath(*read_digest) : storage_.CAS().BlobPath(*read_digest, /*is_executable=*/false); if (not path) { auto const str = fmt::format("could not find {}", read_digest->hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{::grpc::StatusCode::NOT_FOUND, str}; } auto const to_read = IncrementalReader::FromFile(ByteStreamUtils::kChunkSize, *path); if (not to_read.has_value()) { auto const str = fmt::format("Failed to create reader for {}:\n{}", read_digest->hash(), to_read.error()); logger_.Emit(LogLevel::Error, str); return grpc::Status{grpc::StatusCode::INTERNAL, str}; } ::google::bytestream::ReadResponse response; for (auto it = to_read->make_iterator(request->read_offset()); it != to_read->end(); ++it) { auto const chunk = *it; if (not chunk.has_value()) { auto const str = fmt::format("Failed to read data for {}:\n{}", read_digest->hash(), chunk.error()); logger_.Emit(LogLevel::Error, str); return grpc::Status{grpc::StatusCode::INTERNAL, str}; } *response.mutable_data() = *chunk; writer->Write(response); } return ::grpc::Status::OK; } auto BytestreamServiceImpl::Write( ::grpc::ServerContext* /*context*/, ::grpc::ServerReader<::google::bytestream::WriteRequest>* reader, ::google::bytestream::WriteResponse* response) -> ::grpc::Status { ::google::bytestream::WriteRequest request; reader->Read(&request); logger_.Emit( LogLevel::Debug, "Write(resource_name={})", request.resource_name()); auto const write_request = ByteStreamUtils::WriteRequest::FromString(request.resource_name()); if (not write_request) { auto const str = fmt::format("could not parse {}", request.resource_name()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } auto const write_digest = write_request->GetDigest(storage_config_.hash_function.GetType()); if (not write_digest) { logger_.Emit(LogLevel::Debug, "{}", write_digest.error()); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, write_digest.error()}; } logger_.Emit(LogLevel::Debug, [&write_request, &write_digest]() { return fmt::format( "Write(instance_name={}, digest={})", nlohmann::json(write_request->GetInstanceName()).dump(), write_digest->hash()); }); logger_.Emit(LogLevel::Trace, "Write: {}, offset {}, finish write {}", write_digest->hash(), request.write_offset(), request.finish_write()); auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "Could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } auto const tmp_dir = storage_config_.CreateTypedTmpDir("execution-service"); if (not tmp_dir) { return ::grpc::Status{::grpc::StatusCode::INTERNAL, "could not create TmpDir"}; } auto tmp = tmp_dir->GetPath() / write_digest->hash(); { std::ofstream stream{tmp, std::ios::binary}; do { // NOLINT(cppcoreguidelines-avoid-do-while) if (not stream.good()) { auto const str = fmt::format("Failed to write data for {}", write_digest->hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{::grpc::StatusCode::INTERNAL, str}; } stream.write(request.data().data(), static_cast(request.data().size())); } while (not request.finish_write() and reader->Read(&request)); } auto const status = CASUtils::AddFileToCAS(*write_digest, tmp, storage_); if (not status.ok()) { auto const str = fmt::format("Write: {}", status.error_message()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{status.error_code(), str}; } response->set_committed_size( static_cast(std::filesystem::file_size(tmp))); return ::grpc::Status::OK; } auto BytestreamServiceImpl::QueryWriteStatus( ::grpc::ServerContext* /*context*/, const ::google::bytestream::QueryWriteStatusRequest* /*request*/, ::google::bytestream::QueryWriteStatusResponse* /*response*/) -> ::grpc::Status { static constexpr auto kStr = "QueryWriteStatus not implemented"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } bytestream_server.hpp000066400000000000000000000107271516554100600351700ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BYTESTREAM_SERVER_HPP #define BYTESTREAM_SERVER_HPP #include #include "google/bytestream/bytestream.grpc.pb.h" #include "google/bytestream/bytestream.pb.h" #include "gsl/gsl" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" class BytestreamServiceImpl : public ::google::bytestream::ByteStream::Service { public: explicit BytestreamServiceImpl( gsl::not_null const& local_context) noexcept : storage_config_{*local_context->storage_config}, storage_{*local_context->storage} {} // `Read()` is used to retrieve the contents of a resource as a sequence // of bytes. The bytes are returned in a sequence of responses, and the // responses are delivered as the results of a server-side streaming RPC. auto Read(::grpc::ServerContext* context, const ::google::bytestream::ReadRequest* request, ::grpc::ServerWriter<::google::bytestream::ReadResponse>* writer) -> ::grpc::Status override; // `Write()` is used to send the contents of a resource as a sequence of // bytes. The bytes are sent in a sequence of request protos of a // client-side streaming RPC. // // A `Write()` action is resumable. If there is an error or the connection // is broken during the `Write()`, the client should check the status of the // `Write()` by calling `QueryWriteStatus()` and continue writing from the // returned `committed_size`. This may be less than the amount of data the // client previously sent. // // Calling `Write()` on a resource name that was previously written and // finalized could cause an error, depending on whether the underlying // service allows over-writing of previously written resources. // // When the client closes the request channel, the service will respond with // a `WriteResponse`. The service will not view the resource as `complete` // until the client has sent a `WriteRequest` with `finish_write` set to // `true`. Sending any requests on a stream after sending a request with // `finish_write` set to `true` will cause an error. The client **should** // check the `WriteResponse` it receives to determine how much data the // service was able to commit and whether the service views the resource as // `complete` or not. auto Write(::grpc::ServerContext* context, ::grpc::ServerReader<::google::bytestream::WriteRequest>* reader, ::google::bytestream::WriteResponse* response) -> ::grpc::Status override; // `QueryWriteStatus()` is used to find the `committed_size` for a resource // that is being written, which can then be used as the `write_offset` for // the next `Write()` call. // // If the resource does not exist (i.e., the resource has been deleted, or // the first `Write()` has not yet reached the service), this method returns // the error `NOT_FOUND`. // // The client **may** call `QueryWriteStatus()` at any time to determine how // much data has been processed for this resource. This is useful if the // client is buffering data and needs to know which data can be safely // evicted. For any sequence of `QueryWriteStatus()` calls for a given // resource name, the sequence of returned `committed_size` values will be // non-decreasing. auto QueryWriteStatus( ::grpc::ServerContext* context, const ::google::bytestream::QueryWriteStatusRequest* request, ::google::bytestream::QueryWriteStatusResponse* response) -> ::grpc::Status override; private: StorageConfig const& storage_config_; Storage const& storage_; Logger logger_{"execution-service:bytestream"}; }; #endif // BYTESTREAM_SERVER_HPP capabilities_server.cpp000066400000000000000000000043131516554100600354270ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/capabilities_server.hpp" #include "build/bazel/semver/semver.pb.h" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" auto CapabilitiesServiceImpl::GetCapabilities( ::grpc::ServerContext* /*context*/, const ::bazel_re::GetCapabilitiesRequest* /*request*/, ::bazel_re::ServerCapabilities* response) -> ::grpc::Status { ::bazel_re::CacheCapabilities cache; ::bazel_re::ExecutionCapabilities exec; cache.add_digest_functions( ProtocolTraits::IsNative(hash_type_) ? ::bazel_re::DigestFunction_Value::DigestFunction_Value_SHA1 : ::bazel_re::DigestFunction_Value::DigestFunction_Value_SHA256); cache.mutable_action_cache_update_capabilities()->set_update_enabled(false); cache.set_max_batch_total_size_bytes(MessageLimits::kMaxGrpcLength); *(response->mutable_cache_capabilities()) = cache; exec.set_digest_function( ProtocolTraits::IsNative(hash_type_) ? ::bazel_re::DigestFunction_Value::DigestFunction_Value_SHA1 : ::bazel_re::DigestFunction_Value::DigestFunction_Value_SHA256); exec.set_exec_enabled(true); *(response->mutable_execution_capabilities()) = exec; ::build::bazel::semver::SemVer low_v{}; low_v.set_major(2); low_v.set_minor(0); ::build::bazel::semver::SemVer high_v{}; high_v.set_major(2); high_v.set_minor(1); *(response->mutable_low_api_version()) = low_v; *(response->mutable_high_api_version()) = high_v; return ::grpc::Status::OK; } capabilities_server.hpp000066400000000000000000000035251516554100600354400ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef CAPABILITIES_SERVER_HPP #define CAPABILITIES_SERVER_HPP #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" class CapabilitiesServiceImpl final : public bazel_re::Capabilities::Service { public: explicit CapabilitiesServiceImpl(HashFunction::Type hash_type) noexcept : hash_type_{hash_type} {} // GetCapabilities returns the server capabilities configuration of the // remote endpoint. // Only the capabilities of the services supported by the endpoint will // be returned: // * Execution + CAS + Action Cache endpoints should return both // CacheCapabilities and ExecutionCapabilities. // * Execution only endpoints should return ExecutionCapabilities. // * CAS + Action Cache only endpoints should return CacheCapabilities. auto GetCapabilities(::grpc::ServerContext* context, const ::bazel_re::GetCapabilitiesRequest* request, ::bazel_re::ServerCapabilities* response) -> ::grpc::Status override; private: HashFunction::Type const hash_type_; }; #endif // CAPABILITIES_SERVER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/cas_server.cpp000066400000000000000000000336411516554100600336310ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "google/rpc/status.pb.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/expected.hpp" constexpr int kLogBlobLimit = 5; auto CASServiceImpl::FindMissingBlobs( ::grpc::ServerContext* /*context*/, const ::bazel_re::FindMissingBlobsRequest* request, ::bazel_re::FindMissingBlobsResponse* response) -> ::grpc::Status { auto const lock = GarbageCollector::SharedLock(storage_config_); logger_.Emit(LogLevel::Debug, [request]() { std::ostringstream msg{}; msg << "FindMissingBlobs("; msg << "instance_name=" << nlohmann::json(request->instance_name()).dump(); msg << ", blob_digests: "; int count = 0; for (auto const& x : request->blob_digests()) { if (count > 0) { msg << ", "; } if (count > kLogBlobLimit) { msg << "... (" << request->blob_digests_size() << " total)"; break; } msg << x.hash(); count++; } msg << ")"; return msg.str(); }); if (not lock) { static constexpr auto kStr = "FindMissingBlobs: could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } for (auto const& x : request->blob_digests()) { auto const digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), x); bool is_in_cas = false; if (digest) { logger_.Emit( LogLevel::Trace, "FindMissingBlobs: {}", digest->hash()); is_in_cas = digest->IsTree() ? storage_.CAS().TreePath(*digest).has_value() : storage_.CAS().BlobPath(*digest, false).has_value(); } else { logger_.Emit(LogLevel::Error, "FindMissingBlobs: unsupported digest {}", x.hash()); } if (not is_in_cas) { *response->add_missing_blob_digests() = x; } } return ::grpc::Status::OK; } auto CASServiceImpl::BatchUpdateBlobs( ::grpc::ServerContext* /*context*/, const ::bazel_re::BatchUpdateBlobsRequest* request, ::bazel_re::BatchUpdateBlobsResponse* response) -> ::grpc::Status { auto const lock = GarbageCollector::SharedLock(storage_config_); logger_.Emit(LogLevel::Debug, [request]() { std::ostringstream msg{}; msg << "BatchUpdateBlobs("; msg << "instance_name=" << nlohmann::json(request->instance_name()).dump(); msg << ", requests: "; int count = 0; for (auto const& x : request->requests()) { if (count > 0) { msg << ", "; } if (count > kLogBlobLimit) { msg << "... (" << request->requests_size() << " total)"; break; } msg << x.digest().hash(); count++; } msg << ")"; return msg.str(); }); if (not lock) { static constexpr auto kStr = "BatchUpdateBlobs: could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } auto const hash_type = storage_config_.hash_function.GetType(); for (auto const& x : request->requests()) { auto const& hash = x.digest().hash(); logger_.Emit(LogLevel::Trace, "BatchUpdateBlobs: {}", hash); auto const digest = ArtifactDigestFactory::FromBazel(hash_type, x.digest()); if (not digest) { auto const str = fmt::format("BatchUpdateBlobs: unsupported digest {}", hash); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } logger_.Emit(LogLevel::Trace, "BatchUpdateBlobs: {}", digest->hash()); auto* r = response->add_responses(); r->mutable_digest()->CopyFrom(x.digest()); auto const status = CASUtils::AddDataToCAS(*digest, x.data(), storage_); if (not status.ok()) { auto const str = fmt::format("BatchUpdateBlobs: {}", status.error_message()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{status.error_code(), str}; } } return ::grpc::Status::OK; } auto CASServiceImpl::BatchReadBlobs( ::grpc::ServerContext* /*context*/, const ::bazel_re::BatchReadBlobsRequest* request, ::bazel_re::BatchReadBlobsResponse* response) -> ::grpc::Status { static constexpr int kLogBlobLimit = 5; auto const lock = GarbageCollector::SharedLock(storage_config_); logger_.Emit(LogLevel::Debug, [request]() { std::ostringstream msg{}; msg << "BatchReadBlobs("; msg << "instance_name=" << nlohmann::json(request->instance_name()).dump(); msg << ", digests: "; int count = 0; for (auto const& x : request->digests()) { if (count > 0) { msg << ", "; } if (count > kLogBlobLimit) { msg << "... (" << request->digests_size() << " total)"; break; } msg << x.hash(); count++; } msg << ")"; return msg.str(); }); if (not lock) { static constexpr auto kStr = "BatchReadBlobs: Could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } for (auto const& x : request->digests()) { auto* r = response->add_responses(); r->mutable_digest()->CopyFrom(x); auto const digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), x); if (not digest) { auto const str = fmt::format("BatchReadBlobs: unsupported digest {}", x.hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } auto const path = digest->IsTree() ? storage_.CAS().TreePath(*digest) : storage_.CAS().BlobPath(*digest, /*is_executable=*/false); if (not path) { google::rpc::Status status; status.set_code(grpc::StatusCode::NOT_FOUND); r->mutable_status()->CopyFrom(status); continue; } std::ifstream cert{*path}; std::string tmp((std::istreambuf_iterator(cert)), std::istreambuf_iterator()); *(r->mutable_data()) = std::move(tmp); r->mutable_status()->CopyFrom(google::rpc::Status{}); } return ::grpc::Status::OK; } auto CASServiceImpl::GetTree( ::grpc::ServerContext* /*context*/, const ::bazel_re::GetTreeRequest* /*request*/, ::grpc::ServerWriter<::bazel_re::GetTreeResponse>* /*writer*/) -> ::grpc::Status { static constexpr auto kStr = "GetTree not implemented"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/, const ::bazel_re::SplitBlobRequest* request, ::bazel_re::SplitBlobResponse* response) -> ::grpc::Status { logger_.Emit(LogLevel::Debug, [request]() { return fmt::format("SplitBlob(instance_name={}, blob_digest={})", nlohmann::json(request->instance_name()).dump(), request->blob_digest().hash()); }); if (not request->has_blob_digest()) { static constexpr auto kStr = "SplitBlob: no blob digest provided"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, kStr}; } auto const blob_digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), request->blob_digest()); if (not blob_digest) { auto const str = fmt::format("SplitBlob: unsupported digest {}", request->blob_digest().hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } // Acquire garbage collection lock. auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "SplitBlob: could not acquire garbage collection lock"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } // Split blob into chunks. auto const split_result = CASUtils::SplitBlobFastCDC(*blob_digest, storage_); if (not split_result) { auto const& status = split_result.error(); auto const str = fmt::format("SplitBlob: {}", status.error_message()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{status.error_code(), str}; } auto const& chunk_digests = *split_result; logger_.Emit(LogLevel::Debug, [&blob_digest, &chunk_digests]() { std::stringstream ss{}; ss << "Split blob " << blob_digest->hash() << ":" << blob_digest->size() << " into " << chunk_digests.size() << " chunks: [ "; for (auto const& chunk_digest : chunk_digests) { ss << chunk_digest.hash() << ":" << chunk_digest.size() << " "; } ss << "]"; return ss.str(); }); std::transform(chunk_digests.cbegin(), chunk_digests.cend(), pb::back_inserter(response->mutable_chunk_digests()), [](ArtifactDigest const& digest) { return ArtifactDigestFactory::ToBazel(digest); }); return ::grpc::Status::OK; } auto CASServiceImpl::SpliceBlob(::grpc::ServerContext* /*context*/, const ::bazel_re::SpliceBlobRequest* request, ::bazel_re::SpliceBlobResponse* response) -> ::grpc::Status { logger_.Emit(LogLevel::Debug, [request]() { return fmt::format("SplitBlob(instance_name={}, blob_digest={})", nlohmann::json(request->instance_name()).dump(), request->blob_digest().hash()); }); if (not request->has_blob_digest()) { static constexpr auto kStr = "SpliceBlob: no blob digest provided"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, kStr}; } auto const hash_type = storage_config_.hash_function.GetType(); auto const blob_digest = ArtifactDigestFactory::FromBazel(hash_type, request->blob_digest()); if (not blob_digest) { auto const str = fmt::format("SpliceBlob: unsupported digest {}", request->blob_digest().hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } logger_.Emit(LogLevel::Debug, "SpliceBlob({}, {} chunks)", blob_digest->hash(), request->chunk_digests().size()); auto chunk_digests = std::vector{}; chunk_digests.reserve(request->chunk_digests().size()); for (auto const& x : request->chunk_digests()) { auto chunk = ArtifactDigestFactory::FromBazel(hash_type, x); if (not chunk) { auto const str = fmt::format("SpliceBlob: unsupported digest {}", x.hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } chunk_digests.emplace_back(*std::move(chunk)); } // Acquire garbage collection lock. auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "SpliceBlob: could not acquire garbage collection lock"; logger_.Emit(LogLevel::Error, "{}", kStr); return ::grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } // Splice blob from chunks. auto const splice_result = CASUtils::SpliceBlob(*blob_digest, chunk_digests, storage_); if (not splice_result) { auto const& status = splice_result.error(); auto const str = fmt::format("SpliceBlob: {}", status.error_message()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{status.error_code(), str}; } (*response->mutable_blob_digest()) = ArtifactDigestFactory::ToBazel(*splice_result); return ::grpc::Status::OK; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/cas_server.hpp000066400000000000000000000246401516554100600336350ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef CAS_SERVER_HPP #define CAS_SERVER_HPP #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" class CASServiceImpl final : public bazel_re::ContentAddressableStorage::Service { public: explicit CASServiceImpl( gsl::not_null const& local_context) noexcept : storage_config_{*local_context->storage_config}, storage_{*local_context->storage} {} // Determine if blobs are present in the CAS. // // Clients can use this API before uploading blobs to determine which ones // are already present in the CAS and do not need to be uploaded again. // // There are no method-specific errors. auto FindMissingBlobs(::grpc::ServerContext* context, const ::bazel_re::FindMissingBlobsRequest* request, ::bazel_re::FindMissingBlobsResponse* response) -> ::grpc::Status override; // Upload many blobs at once. // // The server may enforce a limit of the combined total size of blobs // to be uploaded using this API. This limit may be obtained using the // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. // Requests exceeding the limit should either be split into smaller // chunks or uploaded using the // [ByteStream API][google.bytestream.ByteStream], as appropriate. // // This request is equivalent to calling a Bytestream `Write` request // on each individual blob, in parallel. The requests may succeed or fail // independently. // // Errors: // // * `INVALID_ARGUMENT`: The client attempted to upload more than the // server supported limit. // // Individual requests may return the following errors, additionally: // // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the // blob. // * `INVALID_ARGUMENT`: The // [Digest][build.bazel.remote.execution.v2.Digest] does not match the // provided data. auto BatchUpdateBlobs(::grpc::ServerContext* context, const ::bazel_re::BatchUpdateBlobsRequest* request, ::bazel_re::BatchUpdateBlobsResponse* response) -> ::grpc::Status override; // Download many blobs at once. // // The server may enforce a limit of the combined total size of blobs // to be downloaded using this API. This limit may be obtained using the // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. // Requests exceeding the limit should either be split into smaller // chunks or downloaded using the // [ByteStream API][google.bytestream.ByteStream], as appropriate. // // This request is equivalent to calling a Bytestream `Read` request // on each individual blob, in parallel. The requests may succeed or fail // independently. // // Errors: // // * `INVALID_ARGUMENT`: The client attempted to read more than the // server supported limit. // // Every error on individual read will be returned in the corresponding // digest status. auto BatchReadBlobs(::grpc::ServerContext* context, const ::bazel_re::BatchReadBlobsRequest* request, ::bazel_re::BatchReadBlobsResponse* response) -> ::grpc::Status override; // Fetch the entire directory tree rooted at a node. // // This request must be targeted at a // [Directory][build.bazel.remote.execution.v2.Directory] stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] // (CAS). The server will enumerate the `Directory` tree recursively and // return every node descended from the root. // // The GetTreeRequest.page_token parameter can be used to skip ahead in // the stream (e.g. when retrying a partially completed and aborted // request), by setting it to a value taken from // GetTreeResponse.next_page_token of the last successfully processed // GetTreeResponse). // // The exact traversal order is unspecified and, unless retrieving // subsequent pages from an earlier request, is not guaranteed to be stable // across multiple invocations of `GetTree`. // // If part of the tree is missing from the CAS, the server will return the // portion present and omit the rest. // // Errors: // // * `NOT_FOUND`: The requested tree root is not present in the CAS. auto GetTree(::grpc::ServerContext* context, const ::bazel_re::GetTreeRequest* request, ::grpc::ServerWriter<::bazel_re::GetTreeResponse>* writer) -> ::grpc::Status override; // Split a blob into chunks. // // This splitting API aims to reduce download traffic between client and // server, e.g., if a client needs to fetch a large blob that just has been // modified slightly since the last built. In this case, there is no need to // fetch the entire blob data, but just the binary differences between the // two blob versions, which are typically determined by deduplication // techniques such as content-defined chunking. // // Clients can use this API before downloading a blob to determine which // parts of the blob are already present locally and do not need to be // downloaded again. The server splits the blob into chunks according to a // specified content-defined chunking algorithm and returns a list of the // chunk digests in the order in which the chunks have to be concatenated to // assemble the requested blob. // // A client can expect the following guarantees from the server if a split // request is answered successfully: // 1. The blob chunks are stored in CAS. // 2. Concatenating the blob chunks in the order of the digest list // returned by the server results in the original blob. // // The usage of this API is optional for clients but it allows them to // download only the missing parts of a large blob instead of the entire // blob data, which in turn can considerably reduce download network // traffic. // // Since the generated chunks are stored as blobs, they underlie the same // lifetimes as other blobs. However, their lifetime is extended if they are // part of the result of a split blob request. // // For the client, it is recommended to verify whether the digest of the // blob assembled by the fetched chunks results in the requested blob // digest. // // If several clients use blob splitting, it is recommended that they // request the same splitting algorithm to benefit from each others chunking // data. In combination with blob splicing, an agreement about the chunking // algorithm is recommended since both client as well as server side can // benefit from each others chunking data. // // Errors: // // * `NOT_FOUND`: The requested blob is not present in the CAS. // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the // blob chunks. auto SplitBlob(::grpc::ServerContext* context, const ::bazel_re::SplitBlobRequest* request, ::bazel_re::SplitBlobResponse* response) -> ::grpc::Status override; // Splice a blob from chunks. // // This is the complementary operation to the // [ContentAddressableStorage.SplitBlob][build.bazel.remote.execution.v2.ContentAddressableStorage.SplitBlob] // function to handle the splitted upload of large blobs to save upload // traffic. // // If a client needs to upload a large blob and is able to split a blob into // chunks locally according to some content-defined chunking algorithm, it // can first determine which parts of the blob are already available in the // remote CAS and upload the missing chunks, and then use this API to // instruct the server to splice the original blob from the remotely // available blob chunks. // // In order to ensure data consistency of the CAS, the server will verify // the spliced result whether digest calculation results in the provided // digest from the request and will reject a splice request if this check // fails. // // The usage of this API is optional for clients but it allows them to // upload only the missing parts of a large blob instead of the entire blob // data, which in turn can considerably reduce upload network traffic. // // In order to split a blob into chunks, it is recommended for the client to // use one of the servers' advertised chunking algorithms by // [CacheCapabilities.supported_chunking_algorithms][build.bazel.remote.execution.v2.CacheCapabilities.supported_chunking_algorithms] // to benefit from each others chunking data. If several clients use blob // splicing, it is recommended that they use the same splitting algorithm to // split their blobs into chunk. // // Errors: // // * `NOT_FOUND`: At least one of the blob chunks is not present in the CAS. // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the // spliced blob. // * `INVALID_ARGUMENT`: The digest of the spliced blob is different from // the provided expected digest. auto SpliceBlob(::grpc::ServerContext* context, const ::bazel_re::SpliceBlobRequest* request, ::bazel_re::SpliceBlobResponse* response) -> ::grpc::Status override; private: StorageConfig const& storage_config_; Storage const& storage_; Logger logger_{"execution-service"}; }; #endif // CAS_SERVER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/cas_utils.cpp000066400000000000000000000156771516554100600334740ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include #include #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/storage/large_object_cas.hpp" namespace { [[nodiscard]] auto ToGrpc(LargeObjectError&& error) noexcept -> grpc::Status { switch (error.Code()) { case LargeObjectErrorCode::Internal: return grpc::Status{grpc::StatusCode::INTERNAL, std::move(error).Message()}; case LargeObjectErrorCode::FileNotFound: return grpc::Status{grpc::StatusCode::NOT_FOUND, std::move(error).Message()}; case LargeObjectErrorCode::InvalidResult: case LargeObjectErrorCode::InvalidTree: return grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, std::move(error).Message()}; } return grpc::Status{grpc::StatusCode::INTERNAL, "an unknown error"}; } class CASContentValidator final { public: explicit CASContentValidator(gsl::not_null const& storage, bool is_owner = true) noexcept; template [[nodiscard]] auto Add(ArtifactDigest const& digest, TData const& data) const noexcept -> grpc::Status { if (digest.IsTree()) { // For trees, check whether the tree invariant holds before storing // the actual tree object. if (auto err = storage_.CAS().CheckTreeInvariant(digest, data)) { return ToGrpc(std::move(*err)); } } auto const cas_digest = digest.IsTree() ? StoreTree(data) : StoreBlob(data); if (not cas_digest) { // This is a serious problem: we have a sequence of bytes, but // cannot write them to CAS. return ::grpc::Status{grpc::StatusCode::INTERNAL, fmt::format("Could not upload {} {}", digest.IsTree() ? "tree" : "blob", digest.hash())}; } if (auto err = CheckDigestConsistency(digest, *cas_digest)) { // User error: did not get a file with the announced hash return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, *std::move(err)}; } return ::grpc::Status::OK; } private: Storage const& storage_; bool const is_owner_; template [[nodiscard]] auto StoreTree(TData const& data) const noexcept -> std::optional { if constexpr (std::is_same_v) { return storage_.CAS().StoreTree(data); } else { return is_owner_ ? storage_.CAS().StoreTree(data) : storage_.CAS().StoreTree(data); } } template [[nodiscard]] auto StoreBlob(TData const& data) const noexcept -> std::optional { static constexpr bool kIsExec = false; if constexpr (std::is_same_v) { return storage_.CAS().StoreBlob(data, kIsExec); } else { return is_owner_ ? storage_.CAS().StoreBlob(data, kIsExec) : storage_.CAS().StoreBlob(data, kIsExec); } } [[nodiscard]] auto CheckDigestConsistency(ArtifactDigest const& ref, ArtifactDigest const& computed) const noexcept -> std::optional; }; } // namespace auto CASUtils::AddDataToCAS(ArtifactDigest const& digest, std::string const& content, Storage const& storage) noexcept -> grpc::Status { return CASContentValidator{&storage}.Add(digest, content); } auto CASUtils::AddFileToCAS(ArtifactDigest const& digest, std::filesystem::path const& file, Storage const& storage, bool is_owner) noexcept -> grpc::Status { return CASContentValidator{&storage, is_owner}.Add(digest, file); } auto CASUtils::SplitBlobFastCDC(ArtifactDigest const& blob_digest, Storage const& storage) noexcept -> expected, grpc::Status> { // Split blob into chunks: auto split = blob_digest.IsTree() ? storage.CAS().SplitTree(blob_digest) : storage.CAS().SplitBlob(blob_digest); if (not split) { return unexpected{ToGrpc(std::move(split).error())}; } return *std::move(split); } auto CASUtils::SpliceBlob(ArtifactDigest const& blob_digest, std::vector const& chunk_digests, Storage const& storage) noexcept -> expected { // Splice blob from chunks: auto splice = blob_digest.IsTree() ? storage.CAS().SpliceTree(blob_digest, chunk_digests) : storage.CAS().SpliceBlob(blob_digest, chunk_digests, false); if (not splice) { return unexpected{ToGrpc(std::move(splice).error())}; } return *std::move(splice); } namespace { CASContentValidator::CASContentValidator( gsl::not_null const& storage, bool is_owner) noexcept : storage_{*storage}, is_owner_{is_owner} {} auto CASContentValidator::CheckDigestConsistency(ArtifactDigest const& ref, ArtifactDigest const& computed) const noexcept -> std::optional { bool valid = ref == computed; if (valid) { bool const check_sizes = not ProtocolTraits::IsNative( storage_.GetHashFunction().GetType()) or ref.size() != 0; if (check_sizes) { valid = ref.size() == computed.size(); } } if (not valid) { return fmt::format( "Expected digest {}:{} and computed digest {}:{} do not match.", ref.hash(), ref.size(), computed.hash(), computed.size()); } return std::nullopt; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/cas_utils.hpp000066400000000000000000000041021516554100600334560ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" class CASUtils { public: [[nodiscard]] static auto AddDataToCAS(ArtifactDigest const& digest, std::string const& content, Storage const& storage) noexcept -> grpc::Status; [[nodiscard]] static auto AddFileToCAS(ArtifactDigest const& digest, std::filesystem::path const& file, Storage const& storage, bool is_owner = true) noexcept -> grpc::Status; [[nodiscard]] static auto SplitBlobFastCDC( ArtifactDigest const& blob_digest, Storage const& storage) noexcept -> expected, grpc::Status>; [[nodiscard]] static auto SpliceBlob( ArtifactDigest const& blob_digest, std::vector const& chunk_digests, Storage const& storage) noexcept -> expected; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP execution_server.cpp000066400000000000000000000543571516554100600350160ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "google/protobuf/any.pb.h" #include "google/protobuf/repeated_ptr_field.h" #include "google/protobuf/timestamp.pb.h" #include "google/rpc/status.pb.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { void SetTimeStamp( gsl::not_null<::google::protobuf::Timestamp*> const& t, std::chrono::time_point const& tvalue) { const int64_t k_nanoseconds_per_second = 1'000'000'000; auto nanos = std::chrono::duration_cast( tvalue.time_since_epoch()) .count(); t->set_seconds(nanos / k_nanoseconds_per_second); t->set_nanos(static_cast(nanos % k_nanoseconds_per_second)); } void UpdateTimeStamp( gsl::not_null<::google::longrunning::Operation*> const& op) { ::google::protobuf::Timestamp t; SetTimeStamp(&t, std::chrono::high_resolution_clock::now()); op->mutable_metadata()->PackFrom(t); } [[nodiscard]] auto ToBazelActionResult( LocalResponse::ArtifactInfos const& artifacts, LocalResponse::DirSymlinks const* dir_symlinks, Storage const& storage, bool legacy_client) noexcept -> expected; [[nodiscard]] auto ToBazelAction(ArtifactDigest const& action_digest, Storage const& storage) noexcept -> expected<::bazel_re::Action, std::string>; [[nodiscard]] auto ToBazelCommand(bazel_re::Action const& action, Storage const& storage) noexcept -> expected; } // namespace auto ExecutionServiceImpl::ToIExecutionAction( ::bazel_re::Action const& action, ::bazel_re::Command const& command, bool legacy_client) const noexcept -> std::optional { auto const root_digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), action.input_root_digest()); if (not root_digest) { return std::nullopt; } std::vector const args(command.arguments().begin(), command.arguments().end()); std::map env_vars; for (auto const& x : command.environment_variables()) { env_vars.insert_or_assign(x.name(), x.value()); } auto execution_action = IExecutionAction::Ptr{}; if (legacy_client) { // force legacy mode, DEPRECATED as of RBEv2.1 std::vector const files(command.output_files().begin(), command.output_files().end()); std::vector const dirs( command.output_directories().begin(), command.output_directories().end()); execution_action = api_.CreateAction(*root_digest, args, command.working_directory(), files, dirs, env_vars, /*properties=*/{}, /*force_legacy=*/true); } else { std::vector const paths(command.output_paths().begin(), command.output_paths().end()); execution_action = api_.CreateAction(*root_digest, args, command.working_directory(), paths, env_vars, /*properties=*/{}); } if (execution_action == nullptr) { return std::nullopt; } execution_action->SetCacheFlag( action.do_not_cache() ? IExecutionAction::CacheFlag::DoNotCacheOutput : IExecutionAction::CacheFlag::CacheOutput); return execution_action; } auto ExecutionServiceImpl::ToBazelExecuteResponse( gsl::not_null const& local_response, bool legacy_client) const noexcept -> expected<::bazel_re::ExecuteResponse, std::string> { auto artifacts = local_response->Artifacts(); if (not artifacts) { return unexpected{std::move(artifacts).error()}; } auto dir_symlinks = local_response->DirectorySymlinks(); LocalResponse::DirSymlinks const* dir_symlinks_ptr{}; if (dir_symlinks) { dir_symlinks_ptr = *dir_symlinks; } else if (legacy_client) { // For legacy clients (ExitCode()); if (local_response->HasStdErr()) { auto const cas_digest = storage_.CAS().StoreBlob(local_response->StdErr(), /*is_executable=*/false); if (not cas_digest) { return unexpected{fmt::format("Could not store stderr of action {}", local_response->ActionDigest())}; } (*action_result.mutable_stderr_digest()) = ArtifactDigestFactory::ToBazel(*cas_digest); } if (local_response->HasStdOut()) { auto const cas_digest = storage_.CAS().StoreBlob(local_response->StdOut(), /*is_executable=*/false); if (not cas_digest) { return unexpected{fmt::format("Could not store stdout of action {}", local_response->ActionDigest())}; } (*action_result.mutable_stdout_digest()) = ArtifactDigestFactory::ToBazel(*cas_digest); } ::bazel_re::ExecuteResponse bazel_response{}; (*bazel_response.mutable_result()) = std::move(action_result); bazel_response.set_cached_result(local_response->IsCached()); // we run the action locally, so no communication issues should happen bazel_response.mutable_status()->set_code(grpc::StatusCode::OK); return bazel_response; } void ExecutionServiceImpl::WriteResponse( ::bazel_re::ExecuteResponse const& execute_response, ::grpc::ServerWriter<::google::longrunning::Operation>* writer, ::google::longrunning::Operation&& op) noexcept { // send response to the client op.mutable_response()->PackFrom(execute_response); op.set_done(true); UpdateTimeStamp(&op); op_cache_.Set(op.name(), op); writer->Write(op); } auto ExecutionServiceImpl::Execute( ::grpc::ServerContext* /*context*/, const ::bazel_re::ExecuteRequest* request, ::grpc::ServerWriter<::google::longrunning::Operation>* writer) -> ::grpc::Status { auto const action_digest = ArtifactDigestFactory::FromBazel( storage_config_.hash_function.GetType(), request->action_digest()); if (not action_digest) { logger_.Emit(LogLevel::Error, "{}", action_digest.error()); return grpc::Status{grpc::StatusCode::INTERNAL, action_digest.error()}; } logger_.Emit(LogLevel::Debug, [request, &action_digest]() { return fmt::format("Execute(instance_name={}, action_digest={})", nlohmann::json(request->instance_name()).dump(), action_digest->hash()); }); auto const lock = GarbageCollector::SharedLock(storage_config_); if (not lock) { static constexpr auto kStr = "Could not acquire SharedLock"; logger_.Emit(LogLevel::Error, "{}", kStr); return grpc::Status{grpc::StatusCode::INTERNAL, kStr}; } auto action = ToBazelAction(*action_digest, storage_); if (not action) { logger_.Emit(LogLevel::Error, "{}", action.error()); return ::grpc::Status{grpc::StatusCode::INTERNAL, std::move(action).error()}; } auto command = ToBazelCommand(*action, storage_); if (not command) { logger_.Emit(LogLevel::Error, "{}", command.error()); return ::grpc::Status{grpc::StatusCode::INTERNAL, std::move(command).error()}; } // If output_paths is empty, the client is using legacy API (output_paths().empty(); auto i_execution_action = ToIExecutionAction(*action, *command, legacy_client); if (not i_execution_action) { auto const str = fmt::format("Could not create action from {}", action_digest->hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } logger_.Emit(LogLevel::Debug, [&action_digest, &command]() { std::vector const args(command->arguments().begin(), command->arguments().end()); return fmt::format("Action {} has command line {}", action_digest->hash(), nlohmann::json(args).dump()); }); logger_.Emit(LogLevel::Info, "Execute {}", action_digest->hash()); // send initial response to the client auto op = ::google::longrunning::Operation{}; auto const& op_name = request->action_digest().hash(); op.set_name(op_name); op.set_done(false); UpdateTimeStamp(&op); op_cache_.Set(op_name, op); writer->Write(op); auto t0 = std::chrono::high_resolution_clock::now(); auto i_execution_response = i_execution_action->get()->Execute(&logger_); auto t1 = std::chrono::high_resolution_clock::now(); logger_.Emit( LogLevel::Trace, "Finished execution of {} in {} seconds", action_digest->hash(), std::chrono::duration_cast(t1 - t0).count()); auto* local_response = dynamic_cast(i_execution_response.get()); if (local_response == nullptr) { auto error_msg = (i_execution_response == nullptr) ? fmt::format("Failed to execute action {}", action_digest->hash()) : std::string{"Local action did not produce a local response"}; logger_.Emit(LogLevel::Error, "{}", error_msg); return ::grpc::Status{grpc::StatusCode::INTERNAL, error_msg}; } auto execute_response = ToBazelExecuteResponse(local_response, legacy_client); if (not execute_response) { logger_.Emit(LogLevel::Error, "{}", execute_response.error()); return ::grpc::Status{grpc::StatusCode::INTERNAL, std::move(execute_response).error()}; } ::bazel_re::ExecuteResponse response = *execute_response; SetTimeStamp(response.mutable_result() ->mutable_execution_metadata() ->mutable_worker_start_timestamp(), t0); SetTimeStamp(response.mutable_result() ->mutable_execution_metadata() ->mutable_worker_completed_timestamp(), t1); // Store the result in action cache if (i_execution_response->ExitCode() == 0 and not action->do_not_cache()) { if (not storage_.ActionCache().StoreResult(*action_digest, response.result())) { auto const str = fmt::format("Could not store action result for action {}", action_digest->hash()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } } WriteResponse(response, writer, std::move(op)); return ::grpc::Status::OK; } auto ExecutionServiceImpl::WaitExecution( ::grpc::ServerContext* /*context*/, const ::bazel_re::WaitExecutionRequest* request, ::grpc::ServerWriter<::google::longrunning::Operation>* writer) -> ::grpc::Status { auto const& hash = request->name(); if (not IsHexString(hash)) { auto const str = fmt::format("Invalid hash {}", hash); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } logger_.Emit(LogLevel::Debug, "WaitExecution: {}", hash); auto op = op_cache_.Query(hash); while (op and not op->done()) { std::this_thread::sleep_for(std::chrono::seconds(1)); op = op_cache_.Query(hash); } if (not op) { auto const str = fmt::format( "Executing action {} not found in internal cache.", hash); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } writer->Write(*op); logger_.Emit(LogLevel::Trace, "Finished WaitExecution {}", hash); return ::grpc::Status::OK; } namespace { [[nodiscard]] auto ToBazelOutputDirectory(std::string path, ArtifactDigest const& digest, Storage const& storage) noexcept -> expected { ::bazel_re::OutputDirectory out_dir{}; *(out_dir.mutable_path()) = std::move(path); LocalCasReader reader(&storage.CAS()); if (ProtocolTraits::IsNative(storage.GetHashFunction().GetType())) { // In native mode: set the digest directly. (*out_dir.mutable_tree_digest()) = ArtifactDigestFactory::ToBazel(digest); } else { // In compatible mode: Create a tree digest from directory // digest on the fly and set tree digest. auto const tree = reader.MakeTree(digest); if (not tree) { return unexpected{fmt::format("Failed to build bazel Tree for {}", digest.hash())}; } auto const cas_digest = storage.CAS().StoreBlob(tree->SerializeAsString(), /*is_executable=*/false); if (not cas_digest) { return unexpected{fmt::format( "Failed to add to the storage the bazel Tree for {}", digest.hash())}; } (*out_dir.mutable_tree_digest()) = ArtifactDigestFactory::ToBazel(*cas_digest); } return out_dir; } [[nodiscard]] auto ToBazelOutputSymlink(std::string path, ArtifactDigest const& digest, Storage const& storage) noexcept -> expected { ::bazel_re::OutputSymlink out_link{}; *(out_link.mutable_path()) = std::move(path); // recover the target of the symlink auto const cas_path = storage.CAS().BlobPath(digest, /*is_executable=*/false); if (not cas_path) { return unexpected{ fmt::format("Failed to recover the symlink for {}", digest.hash())}; } auto content = FileSystemManager::ReadFile(*cas_path); if (not content) { return unexpected{fmt::format( "Failed to read the symlink content for {}", digest.hash())}; } *(out_link.mutable_target()) = *std::move(content); return out_link; } [[nodiscard]] auto ToBazelOutputFile(std::string path, Artifact::ObjectInfo const& info) noexcept -> ::bazel_re::OutputFile { ::bazel_re::OutputFile out_file{}; (*out_file.mutable_path()) = std::move(path); (*out_file.mutable_digest()) = ArtifactDigestFactory::ToBazel(info.digest); out_file.set_is_executable(IsExecutableObject(info.type)); return out_file; } [[nodiscard]] auto ToBazelActionResult( LocalResponse::ArtifactInfos const& artifacts, LocalResponse::DirSymlinks const* dir_symlinks, Storage const& storage, bool legacy_client) noexcept -> expected { if (legacy_client and dir_symlinks == nullptr) { return unexpected{std::string{ "Cannot create action result for clients using RBE protocol (artifacts.size()); result_files.Reserve(size); result_dirs.Reserve(size); if (not legacy_client) { result_links.Reserve(size); } if (dir_symlinks != nullptr) { result_file_links.Reserve(size); result_dir_links.Reserve(size); } for (auto const& [path, info] : artifacts) { if (info.type == ObjectType::Tree) { auto out_dir = ToBazelOutputDirectory(path, info.digest, storage); if (not out_dir) { return unexpected{std::move(out_dir).error()}; } result_dirs.Add(*std::move(out_dir)); } else if (info.type == ObjectType::Symlink) { auto out_link = ToBazelOutputSymlink(path, info.digest, storage); if (not out_link) { return unexpected{std::move(out_link).error()}; } if (not legacy_client) { // only set generic 'output symlinks' if the client was not // using legacy protocol (contains(path)) { // directory symlink result_dir_links.Add(*std::move(out_link)); } else { // file symlinks result_file_links.Add(*std::move(out_link)); } } } else { result_files.Add(ToBazelOutputFile(path, info)); } } return result; } [[nodiscard]] auto ToBazelAction(ArtifactDigest const& action_digest, Storage const& storage) noexcept -> expected<::bazel_re::Action, std::string> { auto const action_path = storage.CAS().BlobPath(action_digest, false); if (not action_path) { return unexpected{fmt::format("could not retrieve blob {} from cas", action_digest.hash())}; } ::bazel_re::Action action{}; if (std::ifstream f(*action_path); not action.ParseFromIstream(&f)) { return unexpected{fmt::format("failed to parse action from blob {}", action_digest.hash())}; } auto const hash_type = storage.GetHashFunction().GetType(); auto const input_root_digest = ArtifactDigestFactory::FromBazel(hash_type, action.input_root_digest()); if (not input_root_digest) { return unexpected{input_root_digest.error()}; } auto const input_root_path = ProtocolTraits::IsNative(hash_type) ? storage.CAS().TreePath(*input_root_digest) : storage.CAS().BlobPath(*input_root_digest, /*is_executable=*/false); if (not input_root_path) { return unexpected{ fmt::format("could not retrieve input root {} from cas", input_root_digest->hash())}; } return action; } [[nodiscard]] auto ToBazelCommand(bazel_re::Action const& action, Storage const& storage) noexcept -> expected { auto const command_digest = ArtifactDigestFactory::FromBazel( storage.GetHashFunction().GetType(), action.command_digest()); if (not command_digest) { return unexpected{command_digest.error()}; } auto const path = storage.CAS().BlobPath(*command_digest, /*is_executable=*/false); if (not path) { return unexpected{fmt::format("Could not retrieve blob {} from cas", command_digest->hash())}; } ::bazel_re::Command c{}; if (std::ifstream f(*path); not c.ParseFromIstream(&f)) { return unexpected{fmt::format("Failed to parse command from blob {}", command_digest->hash())}; } return c; } } // namespace execution_server.hpp000066400000000000000000000170651516554100600350160ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef EXECUTION_SERVER_HPP #define EXECUTION_SERVER_HPP #include #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "google/longrunning/operations.pb.h" #include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" class LocalResponse; class ExecutionServiceImpl final : public bazel_re::Execution::Service { public: explicit ExecutionServiceImpl( gsl::not_null const& local_context, gsl::not_null const& local_api, std::optional op_exponent) noexcept : storage_config_{*local_context->storage_config}, storage_{*local_context->storage}, api_{*local_api} { if (op_exponent) { op_cache_.SetExponent(*op_exponent); } } [[nodiscard]] auto GetOpCache() const noexcept -> OperationCache const& { return op_cache_; } // Execute an action remotely. // // In order to execute an action, the client must first upload all of the // inputs, the // [Command][build.bazel.remote.execution.v2.Command] to run, and the // [Action][build.bazel.remote.execution.v2.Action] into the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. // It then calls `Execute` with an `action_digest` referring to them. The // server will run the action and eventually return the result. // // The input `Action`'s fields MUST meet the various canonicalization // requirements specified in the documentation for their types so that it // has the same digest as other logically equivalent `Action`s. The server // MAY enforce the requirements and return errors if a non-canonical input // is received. It MAY also proceed without verifying some or all of the // requirements, such as for performance reasons. If the server does not // verify the requirement, then it will treat the `Action` as distinct from // another logically equivalent action if they hash differently. // // Returns a stream of // [google.longrunning.Operation][google.longrunning.Operation] messages // describing the resulting execution, with eventual `response` // [ExecuteResponse][build.bazel.remote.execution.v2.ExecuteResponse]. The // `metadata` on the operation is of type // [ExecuteOperationMetadata][build.bazel.remote.execution.v2.ExecuteOperationMetadata]. // // If the client remains connected after the first response is returned // after the server, then updates are streamed as if the client had called // [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution] // until the execution completes or the request reaches an error. The // operation can also be queried using [Operations // API][google.longrunning.Operations.GetOperation]. // // The server NEED NOT implement other methods or functionality of the // Operations API. // // Errors discovered during creation of the `Operation` will be reported // as gRPC Status errors, while errors that occurred while running the // action will be reported in the `status` field of the `ExecuteResponse`. // The server MUST NOT set the `error` field of the `Operation` proto. The // possible errors include: // // * `INVALID_ARGUMENT`: One or more arguments are invalid. // * `FAILED_PRECONDITION`: One or more errors occurred in setting up the // action requested, such as a missing input or command or no worker being // available. The client may be able to fix the errors and retry. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to // run // the action. // * `UNAVAILABLE`: Due to a transient condition, such as all workers being // occupied (and the server does not support a queue), the action could // not be started. The client should retry. // * `INTERNAL`: An internal error occurred in the execution engine or the // worker. // * `DEADLINE_EXCEEDED`: The execution timed out. // * `CANCELLED`: The operation was cancelled by the client. This status is // only possible if the server implements the Operations API // CancelOperation method, and it was called for the current execution. // // In the case of a missing input or command, the server SHOULD additionally // send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail // where, for each requested blob not present in the CAS, there is a // `Violation` with a `type` of `MISSING` and a `subject` of // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. auto Execute(::grpc::ServerContext* context, const ::bazel_re::ExecuteRequest* request, ::grpc::ServerWriter<::google::longrunning::Operation>* writer) -> ::grpc::Status override; // Wait for an execution operation to complete. When the client initially // makes the request, the server immediately responds with the current // status of the execution. The server will leave the request stream open // until the operation completes, and then respond with the completed // operation. The server MAY choose to stream additional updates as // execution progresses, such as to provide an update as to the state of the // execution. auto WaitExecution(::grpc::ServerContext* context, const ::bazel_re::WaitExecutionRequest* request, ::grpc::ServerWriter<::google::longrunning::Operation>* writer) -> ::grpc::Status override; private: StorageConfig const& storage_config_; Storage const& storage_; LocalApi const& api_; OperationCache op_cache_; Logger logger_{"execution-service"}; [[nodiscard]] auto ToIExecutionAction(::bazel_re::Action const& action, ::bazel_re::Command const& command, bool legacy_client) const noexcept -> std::optional; [[nodiscard]] auto ToBazelExecuteResponse( gsl::not_null const& local_response, bool legacy_client) const noexcept -> expected<::bazel_re::ExecuteResponse, std::string>; void WriteResponse( ::bazel_re::ExecuteResponse const& execute_response, ::grpc::ServerWriter<::google::longrunning::Operation>* writer, ::google::longrunning::Operation&& op) noexcept; }; #endif // EXECUTION_SERVER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/operation_cache.cpp000066400000000000000000000033751516554100600346210ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include #include // for operator< #include // for back_insert_iterator #include // for vector #include "google/protobuf/timestamp.pb.h" void OperationCache::GarbageCollection() { if (cache_.size() > (threshold_ << 1U)) { std::vector> tmp; tmp.reserve(cache_.size()); std::copy(cache_.begin(), cache_.end(), std::back_insert_iterator(tmp)); std::sort(tmp.begin(), tmp.end(), [](auto const& x, auto const& y) { ::google::protobuf::Timestamp tx; ::google::protobuf::Timestamp ty; x.second.metadata().UnpackTo(&tx); y.second.metadata().UnpackTo(&ty); return tx.seconds() < ty.seconds(); }); std::size_t deleted = 0; for (auto const& [key, op] : tmp) { if (op.done()) { DropInternal(key); ++deleted; } if (deleted == threshold_) { break; } } } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service/operation_cache.hpp000066400000000000000000000047441516554100600346270ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef OPERATION_CACHE_HPP #define OPERATION_CACHE_HPP #include #include #include #include #include #include #include #include #include #include "google/longrunning/operations.pb.h" class OperationCache final { using Operation = ::google::longrunning::Operation; public: OperationCache() noexcept = default; ~OperationCache() noexcept = default; OperationCache(OperationCache const&) = delete; auto operator=(OperationCache const&) -> OperationCache& = delete; OperationCache(OperationCache&&) = delete; auto operator=(OperationCache&&) -> OperationCache& = delete; void Set(std::string const& action, Operation const& op) { SetInternal(action, op); } [[nodiscard]] auto Query(std::string const& x) const noexcept -> std::optional { return QueryInternal(x); } void SetExponent(std::uint8_t x) noexcept { threshold_ = 1U << x; } private: mutable std::shared_mutex mutex_; std::unordered_map cache_; static constexpr std::uint8_t kDefaultExponent{14}; std::size_t threshold_{1U << kDefaultExponent}; void SetInternal(std::string const& action, Operation const& op) { std::unique_lock lock{mutex_}; GarbageCollection(); cache_[action] = op; } [[nodiscard]] auto QueryInternal(std::string const& x) const noexcept -> std::optional { std::shared_lock lock{mutex_}; auto it = cache_.find(x); if (it != cache_.end()) { return it->second; } return std::nullopt; } void DropInternal(std::string const& x) noexcept { cache_[x].Clear(); cache_.erase(x); } void GarbageCollection(); }; #endif // OPERATION_CACHE_HPP operations_server.cpp000066400000000000000000000057521516554100600351710ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/operations_server.hpp" #include #include #include #include "fmt/core.h" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/hex_string.hpp" auto OperationsServiceImpl::GetOperation( ::grpc::ServerContext* /*context*/, const ::google::longrunning::GetOperationRequest* request, ::google::longrunning::Operation* response) -> ::grpc::Status { auto const& hash = request->name(); if (not IsHexString(hash)) { auto const str = fmt::format("Invalid hash {}", hash); logger_.Emit(LogLevel::Debug, "{}", str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } logger_.Emit(LogLevel::Trace, "GetOperation: {}", hash); auto op = op_cache_.Query(hash); if (not op) { auto const str = fmt::format( "Executing action {} not found in internal cache.", hash); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } *response = *std::move(op); return ::grpc::Status::OK; } auto OperationsServiceImpl::ListOperations( ::grpc::ServerContext* /*context*/, const ::google::longrunning::ListOperationsRequest* /*request*/, ::google::longrunning::ListOperationsResponse* /*response*/) -> ::grpc::Status { static constexpr auto kStr = "ListOperations not implemented"; logger_.Emit(LogLevel::Error, kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } auto OperationsServiceImpl::DeleteOperation( ::grpc::ServerContext* /*context*/, const ::google::longrunning::DeleteOperationRequest* /*request*/, ::google::protobuf::Empty* /*response*/) -> ::grpc::Status { static constexpr auto kStr = "DeleteOperation not implemented"; logger_.Emit(LogLevel::Error, kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } auto OperationsServiceImpl::CancelOperation( ::grpc::ServerContext* /*context*/, const ::google::longrunning::CancelOperationRequest* /*request*/, ::google::protobuf::Empty* /*response*/) -> ::grpc::Status { static constexpr auto kStr = "CancelOperation not implemented"; logger_.Emit(LogLevel::Error, kStr); return ::grpc::Status{grpc::StatusCode::UNIMPLEMENTED, kStr}; } operations_server.hpp000066400000000000000000000073251516554100600351740ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef OPERATIONS_SERVER_HPP #define OPERATIONS_SERVER_HPP #include #include "google/longrunning/operations.grpc.pb.h" #include "google/longrunning/operations.pb.h" #include "google/protobuf/empty.pb.h" #include "gsl/gsl" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/logging/logger.hpp" class OperationsServiceImpl final : public ::google::longrunning::Operations::Service { public: explicit OperationsServiceImpl( gsl::not_null const& op_cache) : op_cache_{*op_cache} {}; // Lists operations that match the specified filter in the request. If the // server doesn't support this method, it returns `UNIMPLEMENTED`. // // NOTE: the `name` binding below allows API services to override the // binding to use different resource name schemes, such as // `users/*/operations`. auto ListOperations( ::grpc::ServerContext* context, const ::google::longrunning::ListOperationsRequest* request, ::google::longrunning::ListOperationsResponse* response) -> ::grpc::Status override; // Gets the latest state of a long-running operation. Clients can use this // method to poll the operation result at intervals as recommended by the // API service. auto GetOperation(::grpc::ServerContext* context, const ::google::longrunning::GetOperationRequest* request, ::google::longrunning::Operation* response) -> ::grpc::Status override; // Deletes a long-running operation. This method indicates that the client // is no longer interested in the operation result. It does not cancel the // operation. If the server doesn't support this method, it returns // `google.rpc.Code.UNIMPLEMENTED`. auto DeleteOperation( ::grpc::ServerContext* context, const ::google::longrunning::DeleteOperationRequest* request, ::google::protobuf::Empty* response) -> ::grpc::Status override; // Starts asynchronous cancellation on a long-running operation. The server // makes a best effort to cancel the operation, but success is not // guaranteed. If the server doesn't support this method, it returns // `google.rpc.Code.UNIMPLEMENTED`. Clients can use // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or // other methods to check whether the cancellation succeeded or whether the // operation completed despite cancellation. On successful cancellation, // the operation is not deleted; instead, it becomes an operation with // an [Operation.error][google.longrunning.Operation.error] value with a // [google.rpc.Status.code][google.rpc.Status.code] of 1, corresponding to // `Code.CANCELLED`. auto CancelOperation( ::grpc::ServerContext* context, const ::google::longrunning::CancelOperationRequest* request, ::google::protobuf::Empty* response) -> ::grpc::Status override; private: OperationCache const& op_cache_; Logger logger_{"execution-service:operations"}; }; #endif // OPERATIONS_SERVER_HPP server_implementation.cpp000066400000000000000000000124541516554100600360300ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/server_implementation.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/execution_service/ac_server.hpp" #include "src/buildtool/execution_api/execution_service/bytestream_server.hpp" #include "src/buildtool/execution_api/execution_service/capabilities_server.hpp" #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include "src/buildtool/execution_api/execution_service/operations_server.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/file_system/atomic.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/type_safe_arithmetic.hpp" auto ServerImpl::Create(std::optional interface, std::optional port, std::optional info_file, std::optional pid_file) noexcept -> std::optional { ServerImpl server; if (interface) { server.interface_ = std::move(*interface); } if (port) { auto parsed_port = ParsePort(*port); if (parsed_port) { server.port_ = static_cast(*parsed_port); } else { Logger::Log(LogLevel::Error, "Invalid port '{}'", *port); return std::nullopt; } } if (info_file) { server.info_file_ = std::move(*info_file); } if (pid_file) { server.pid_file_ = std::move(*pid_file); } return server; } auto ServerImpl::Run(gsl::not_null const& local_context, gsl::not_null const& remote_context, gsl::not_null const& local_api, std::optional op_exponent) -> bool { auto const hash_type = local_context->storage_config->hash_function.GetType(); ExecutionServiceImpl es{local_context, local_api, op_exponent}; ActionCacheServiceImpl ac{local_context}; CASServiceImpl cas{local_context}; BytestreamServiceImpl b{local_context}; CapabilitiesServiceImpl cap{hash_type}; OperationsServiceImpl op{&es.GetOpCache()}; grpc::ServerBuilder builder; builder.RegisterService(&es) .RegisterService(&ac) .RegisterService(&cas) .RegisterService(&b) .RegisterService(&cap) .RegisterService(&op); // check authentication credentials; currently only TLS/SSL is supported std::shared_ptr creds; if (const auto* tls_auth = std::get_if(&remote_context->auth->method); tls_auth != nullptr) { auto tls_opts = grpc::SslServerCredentialsOptions{}; tls_opts.pem_root_certs = tls_auth->ca_cert; grpc::SslServerCredentialsOptions::PemKeyCertPair keycert = { tls_auth->server_key, tls_auth->server_cert}; tls_opts.pem_key_cert_pairs.emplace_back(keycert); creds = grpc::SslServerCredentials(tls_opts); } else { creds = grpc::InsecureServerCredentials(); } builder.AddListeningPort( fmt::format("{}:{}", interface_, port_), creds, &port_); auto server = builder.BuildAndStart(); if (not server) { Logger::Log(LogLevel::Error, "Could not start execution service"); return false; } auto pid = getpid(); nlohmann::json const& info = { {"interface", interface_}, {"port", port_}, {"pid", pid}}; if (not pid_file_.empty()) { if (not FileSystemAtomic::WriteFile(pid_file_, fmt::format("{}", pid))) { server->Shutdown(); return false; } } auto const info_str = nlohmann::to_string(info); Logger::Log(LogLevel::Info, "{}execution service started: {}", ProtocolTraits::IsNative(hash_type) ? "" : "compatible ", info_str); if (not info_file_.empty()) { if (not FileSystemAtomic::WriteFile(info_file_, info_str)) { server->Shutdown(); return false; } } server->Wait(); return true; } server_implementation.hpp000066400000000000000000000042541516554100600360340ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/execution_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef SERVER_IMPLEMENATION_HPP #define SERVER_IMPLEMENATION_HPP #include #include #include #include "gsl/gsl" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/context.hpp" class LocalApi; class ServerImpl final { public: [[nodiscard]] static auto Create( std::optional interface, std::optional port, std::optional info_file, std::optional pid_file) noexcept -> std::optional; ~ServerImpl() noexcept = default; ServerImpl(ServerImpl const&) = delete; auto operator=(ServerImpl const&) noexcept -> ServerImpl& = delete; ServerImpl(ServerImpl&&) noexcept = default; auto operator=(ServerImpl&&) noexcept -> ServerImpl& = default; /// \brief Start the execution service. /// \param local_context The LocalContext to be used. /// \param remote_context The RemoteContext to be used. /// \param local_api The LocalApi used. /// \param op_exponent Log2 threshold for operation cache. auto Run(gsl::not_null const& local_context, gsl::not_null const& remote_context, gsl::not_null const& local_api, std::optional op_exponent) -> bool; private: ServerImpl() noexcept = default; std::string interface_{"127.0.0.1"}; int port_{0}; std::string info_file_; std::string pid_file_; }; #endif // SERVER_IMPLEMENATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/git/000077500000000000000000000000001516554100600260225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/git/TARGETS000066400000000000000000000016251516554100600270620ustar00rootroot00000000000000{ "git_api": { "type": ["@", "rules", "CC", "library"] , "name": ["git_api"] , "hdrs": ["git_api.hpp"] , "srcs": ["git_api.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/common", "common"] ] , "private-deps": [ ["@", "json", "", "json"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_tree"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "execution_api", "git"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/git/git_api.cpp000066400000000000000000000235211516554100600301450ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/git/git_api.hpp" #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" namespace { [[nodiscard]] auto ToArtifactDigest(GitTreeEntry const& entry) noexcept -> std::optional { auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, entry.Hash(), /*size=*/0, entry.IsTree()); if (not digest) { return std::nullopt; } return *std::move(digest); } } // namespace GitApi::GitApi(gsl::not_null const& repo_config) : repo_config_{*repo_config} {} auto GitApi::RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths) const noexcept -> bool { if (artifacts_info.size() != output_paths.size()) { Logger::Log(LogLevel::Error, "different number of digests and output paths."); return false; } for (std::size_t i{}; i < artifacts_info.size(); ++i) { auto const& info = artifacts_info[i]; if (IsTreeObject(info.type)) { auto tree = repo_config_.ReadTreeFromGitCAS(info.digest.hash()); if (not tree) { return false; } for (auto const& [path, entry] : *tree) { auto digest = ToArtifactDigest(*entry); if (not digest or not RetrieveToPaths( {Artifact::ObjectInfo{.digest = *std::move(digest), .type = entry->Type(), .failed = false}}, {output_paths[i] / path})) { return false; } } } else { auto blob = repo_config_.ReadBlobFromGitCAS( info.digest.hash(), /*is_symlink=*/IsSymlinkObject(info.type)); if (not blob) { return false; } if (not FileSystemManager::CreateDirectory( output_paths[i].parent_path()) or not FileSystemManager::WriteFileAs( *blob, output_paths[i], info.type)) { Logger::Log(LogLevel::Error, "staging to output path {} failed.", output_paths[i].string()); return false; } } } return true; } auto GitApi::RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree) const noexcept -> bool { if (artifacts_info.size() != fds.size()) { Logger::Log(LogLevel::Error, "different number of digests and file descriptors."); return false; } for (std::size_t i{}; i < artifacts_info.size(); ++i) { auto const& info = artifacts_info[i]; std::string content; if (IsTreeObject(info.type) and not raw_tree) { auto tree = repo_config_.ReadTreeFromGitCAS(info.digest.hash()); if (not tree) { Logger::Log(LogLevel::Debug, "Tree {} not known to git", info.digest.hash()); return false; } try { auto json = nlohmann::json::object(); for (auto const& [path, entry] : *tree) { auto digest = ToArtifactDigest(*entry); if (not digest) { return false; } json[path] = Artifact::ObjectInfo{.digest = *std::move(digest), .type = entry->Type(), .failed = false} .ToString(/*size_unknown*/ true); } content = json.dump(2) + "\n"; } catch (...) { return false; } } else { auto blob = repo_config_.ReadBlobFromGitCAS( info.digest.hash(), /*is_symlink=*/IsSymlinkObject(info.type)); if (not blob) { Logger::Log(LogLevel::Debug, "Blob {} not known to git", info.digest.hash()); return false; } content = *std::move(blob); } if (gsl::owner out = fdopen(fds[i], "wb")) { // NOLINT std::fwrite(content.data(), 1, content.size(), out); std::fclose(out); } else { Logger::Log(LogLevel::Error, "dumping to file descriptor {} failed.", fds[i]); return false; } } return true; } auto GitApi::RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool { // Determine missing artifacts in api. std::unordered_set> missing; missing.reserve(artifacts_info.size()); { auto back_map = BackMap::Make( &artifacts_info, [](Artifact::ObjectInfo const& info) { return info.digest; }); if (back_map == nullptr) { Logger::Log(LogLevel::Error, "GitApi: Failed to create BackMap"); return false; } auto missing_digests = api.GetMissingDigests(back_map->GetKeys()); missing = back_map->GetReferences(missing_digests); } // GitApi works in the native mode only. HashFunction const hash_function{HashFunction::Type::GitSHA1}; // Collect blobs of missing artifacts from local CAS. Trees are // processed recursively before any blob is uploaded. std::unordered_set container; for (auto const& info : missing) { std::optional content; // Recursively process trees. if (IsTreeObject(info->type)) { auto tree = repo_config_.ReadTreeFromGitCAS(info->digest.hash()); if (not tree) { return false; } std::vector subentries; subentries.reserve(std::distance(tree->begin(), tree->end())); for (auto const& [path, entry] : *tree) { auto digest = ToArtifactDigest(*entry); if (not digest.has_value()) { return false; } subentries.push_back( Artifact::ObjectInfo{.digest = *std::move(digest), .type = entry->Type(), .failed = false}); } if (not RetrieveToCas(subentries, api)) { return false; } content = tree->RawData(); } else { content = repo_config_.ReadBlobFromGitCAS( info->digest.hash(), /*is_symlink=*/IsSymlinkObject(info->type)); } if (not content) { return false; } auto blob = ArtifactBlob::FromMemory( hash_function, info->type, *std::move(content)); if (not blob.has_value()) { return false; } // Collect blob and upload to remote CAS if transfer size reached. if (not UpdateContainerAndUpload( &container, *std::move(blob), /*exception_is_fatal=*/true, [&api](std::unordered_set&& blobs) { return api.Upload(std::move(blobs), /*skip_find_missing=*/true); })) { return false; } } // Upload remaining blobs to remote CAS. return api.Upload(std::move(container), /*skip_find_missing=*/true); } auto GitApi::RetrieveToMemory(Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional { return repo_config_.ReadBlobFromGitCAS( artifact_info.digest.hash(), /*is_symlink=*/IsSymlinkObject(artifact_info.type)); } auto GitApi::IsAvailable(ArtifactDigest const& digest) const noexcept -> bool { return repo_config_ .ReadBlobFromGitCAS( digest.hash(), /*is_symlink=*/false, LogLevel::Trace) .has_value(); // allow invalid symlink if just to check existence } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/git/git_api.hpp000066400000000000000000000056321516554100600301550ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_GIT_GIT_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_GIT_GIT_API_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" class GitApi final { public: explicit GitApi(gsl::not_null const& repo_config); /// \brief Retrieve artifacts from git and store to specified paths. /// Tree artifacts are resolved and its containing file artifacts are /// recursively retrieved. [[nodiscard]] auto RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths) const noexcept -> bool; /// \brief Retrieve artifacts from git and write to file descriptors. /// Tree artifacts are not resolved and instead the tree object will be /// pretty-printed before writing to fd. If `raw_tree` is set, pretty /// printing will be omitted and the raw tree object will be written /// instead. [[nodiscard]] auto RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree) const noexcept -> bool; /// \brief Synchronization of artifacts between api and git. Retrieves /// artifacts from git and uploads to api. Tree artifacts are resolved and /// its containing file artifacts are recursively retrieved. [[nodiscard]] auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool; /// \brief Retrieve one artifact from git and make it available for further /// in-memory processing [[nodiscard]] auto RetrieveToMemory( Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional; /// \brief Check if the given digest is available in git. [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool; private: RepositoryConfig const& repo_config_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_GIT_GIT_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/000077500000000000000000000000001516554100600263315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/TARGETS000066400000000000000000000047321516554100600273730ustar00rootroot00000000000000{ "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "hdrs": ["config.hpp"] , "deps": [["@", "fmt", "", "fmt"], ["src/utils/cpp", "expected"]] , "stage": ["src", "buildtool", "execution_api", "local"] } , "local_api": { "type": ["@", "rules", "CC", "library"] , "name": ["local_api"] , "hdrs": [ "local_api.hpp" , "local_action.hpp" , "local_response.hpp" , "local_cas_reader.hpp" ] , "srcs": ["local_api.cpp", "local_action.cpp", "local_cas_reader.cpp"] , "deps": [ "context" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/git", "git_api"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_api", "local"] , "private-deps": [ "config" , ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/execution_api/execution_service", "cas_utils"] , ["src/buildtool/execution_api/utils", "outputscheck"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/system", "system_command"] , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "incremental_reader"] ] } , "context": { "type": ["@", "rules", "CC", "library"] , "name": ["context"] , "hdrs": ["context.hpp"] , "deps": [ "config" , ["@", "gsl", "", "gsl"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "execution_api", "local"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/config.hpp000066400000000000000000000044561516554100600303200ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/utils/cpp/expected.hpp" /// \brief Store local execution configuration. struct LocalExecutionConfig final { class Builder; // Launcher to be prepended to action's command before executed. // Default: ["env", "--"] std::vector const launcher = {"env", "--"}; }; class LocalExecutionConfig::Builder final { public: auto SetLauncher(std::vector launcher) noexcept -> Builder& { launcher_ = std::move(launcher); return *this; } /// \brief Finalize building and create LocalExecutionConfig. /// \return LocalExecutionConfig on success, an error string on failure. [[nodiscard]] auto Build() const noexcept -> expected { // To not duplicate default arguments in builder, create a default // config and copy arguments from there. LocalExecutionConfig const default_config; auto launcher = default_config.launcher; if (launcher_.has_value()) { try { launcher = *launcher_; } catch (std::exception const& ex) { return unexpected{ fmt::format("Failed to set launcher:\n{}", ex.what())}; } } return LocalExecutionConfig{.launcher = std::move(launcher)}; } private: std::optional> launcher_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/context.hpp000066400000000000000000000024341516554100600305310ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONTEXT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONTEXT_HPP #include "gsl/gsl" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" /// \brief Aggregate of storage entities. /// \note No field is be stored as const ref to avoid binding to temporaries. struct LocalContext final { gsl::not_null const exec_config; gsl::not_null const storage_config; gsl::not_null const storage; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONTEXT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_action.cpp000066400000000000000000000636001516554100600314710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/local/local_action.hpp" #include #include #include #include #include #include #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/execution_api/utils/outputscheck.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/path.hpp" namespace { /// \brief Removes specified directory if KeepBuildDir() is not set. class BuildCleanupAnchor { public: explicit BuildCleanupAnchor(std::filesystem::path build_path) noexcept : build_path_{std::move(build_path)} {} BuildCleanupAnchor(BuildCleanupAnchor const&) = delete; BuildCleanupAnchor(BuildCleanupAnchor&&) = delete; auto operator=(BuildCleanupAnchor const&) -> BuildCleanupAnchor& = delete; auto operator=(BuildCleanupAnchor&&) -> BuildCleanupAnchor& = delete; ~BuildCleanupAnchor() { if (not FileSystemManager::RemoveDirectory(build_path_)) { Logger::Log(LogLevel::Error, "Could not cleanup build directory {}", build_path_.string()); } } private: std::filesystem::path const build_path_; }; [[nodiscard]] auto CreateDigestFromLocalOwnedTree( Storage const& storage, std::filesystem::path const& dir_path) -> std::optional { auto const& cas = storage.CAS(); auto store_blob = [&cas](std::filesystem::path const& path, auto is_exec) -> std::optional { return cas.StoreBlob(path, is_exec); }; auto store_tree = [&cas](std::string const& content) -> std::optional { return cas.StoreTree(content); }; auto store_symlink = [&cas](std::string const& content) -> std::optional { return cas.StoreBlob(content); }; return ProtocolTraits::IsNative(storage.GetHashFunction().GetType()) ? BazelMsgFactory::CreateGitTreeDigestFromLocalTree( dir_path, store_blob, store_tree, store_symlink) : BazelMsgFactory::CreateDirectoryDigestFromLocalTree( dir_path, store_blob, store_tree, store_symlink); } } // namespace auto LocalAction::Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr { auto do_cache = CacheEnabled(cache_flag_); auto const action = CreateActionDigest(root_digest_, not do_cache); if (not action) { if (logger != nullptr) { logger->Emit(LogLevel::Error, "failed to create an action digest for {}", root_digest_.hash()); } return nullptr; } if (logger != nullptr) { logger->Emit(LogLevel::Trace, "start execution\n" " - exec_dir digest: {}\n" " - action digest: {}", root_digest_.hash(), action->hash()); } auto create_response = [](Logger const* logger, std::string const& action_hash, auto&&... args) -> IExecutionResponse::Ptr { try { return IExecutionResponse::Ptr{new LocalResponse{ action_hash, std::forward(args)...}}; } catch (...) { if (logger != nullptr) { logger->Emit(LogLevel::Error, "failed to create a response for {}", action_hash); } } return nullptr; }; if (do_cache) { if (auto result = local_context_.storage->ActionCache().CachedResult(*action)) { if (result->exit_code() == 0 and (mode_ == RequestMode::kV2_1 ? ActionResultContainsExpectedOutputs(*result, output_paths_) : ActionResultContainsExpectedOutputs( *result, output_files_, output_dirs_))) { return create_response( logger, action->hash(), LocalAction::Output{*std::move(result), /*is_cached=*/true}, local_context_.storage); } } } if (ExecutionEnabled(cache_flag_)) { if (auto output = Run(*action)) { if (cache_flag_ == CacheFlag::PretendCached) { // ensure the same id is created as if caching were enabled auto const action_cached = CreateActionDigest(root_digest_, false); if (not action_cached) { if (logger != nullptr) { logger->Emit( LogLevel::Error, "failed to create a cached action digest for {}", root_digest_.hash()); } return nullptr; } output->is_cached = true; return create_response(logger, action_cached->hash(), *std::move(output), local_context_.storage); } return create_response(logger, action->hash(), *std::move(output), local_context_.storage); } } return nullptr; } auto LocalAction::Run(ArtifactDigest const& action_id) const noexcept -> std::optional { auto const exec_path = CreateUniquePath( local_context_.storage_config->ExecutionRoot() / action_id.hash()); if (not exec_path) { return std::nullopt; } // anchor for cleaning up build directory at end of function (using RAII) auto anchor = BuildCleanupAnchor(*exec_path); auto const build_root = *exec_path / "build_root"; if (not CreateDirectoryStructure(build_root)) { return std::nullopt; } if (cmdline_.empty()) { logger_.Emit(LogLevel::Error, "malformed command line"); return std::nullopt; } // prepare actual command by including the launcher auto cmdline = local_context_.exec_config->launcher; std::copy(cmdline_.begin(), cmdline_.end(), std::back_inserter(cmdline)); SystemCommand system{"LocalExecution"}; auto start_time = std::chrono::system_clock::now(); auto const exit_code = system.Execute(cmdline, env_vars_, build_root / cwd_, *exec_path); auto end_time = std::chrono::system_clock::now(); if (exit_code.has_value()) { Output result{}; result.action.set_exit_code(*exit_code); if (auto const digest = DigestFromOwnedFile(*exec_path / "stdout")) { *result.action.mutable_stdout_digest() = ArtifactDigestFactory::ToBazel(*digest); } if (auto const digest = DigestFromOwnedFile(*exec_path / "stderr")) { *result.action.mutable_stderr_digest() = ArtifactDigestFactory::ToBazel(*digest); } if (CollectAndStoreOutputs(&result.action, build_root / cwd_)) { if (cache_flag_ == CacheFlag::CacheOutput) { if (not local_context_.storage->ActionCache().StoreResult( action_id, result.action)) { logger_.Emit(LogLevel::Warning, "failed to store action results"); } } } result.duration = std::chrono::duration(end_time - start_time).count(); return result; } logger_.Emit(LogLevel::Error, "failed to execute commands"); return std::nullopt; } auto LocalAction::StageInput( std::filesystem::path const& target_path, Artifact::ObjectInfo const& info, gsl::not_null copies) const noexcept -> bool { static std::string const kCopyFileName{"blob"}; if (IsTreeObject(info.type)) { return FileSystemManager::CreateDirectory(target_path); } std::optional blob_path{}; if (auto lookup = copies->find(info); lookup != copies->end()) { blob_path = lookup->second->GetPath() / kCopyFileName; } else { blob_path = local_context_.storage->CAS().BlobPath( info.digest, IsExecutableObject(info.type)); } if (not blob_path) { logger_.Emit(LogLevel::Error, "artifact with id {} is missing in CAS", info.digest.hash()); return false; } if (info.type == ObjectType::Symlink) { auto to = FileSystemManager::ReadContentAtPath(*blob_path, ObjectType::File); if (not to) { logger_.Emit(LogLevel::Error, "could not read content of symlink {}", (*blob_path).string()); return false; } return FileSystemManager::CreateSymlink(*to, target_path); } if (not FileSystemManager::CreateDirectory(target_path.parent_path())) { return false; } auto res = FileSystemManager::CreateFileHardlink( *blob_path, target_path, LogLevel::Debug); if (res) { return true; } if (res.error() != std::errc::too_many_links) { logger_.Emit(LogLevel::Warning, "Failed to link {} to {}: {}, {}", nlohmann::json(blob_path->string()).dump(), nlohmann::json(target_path.string()).dump(), res.error().value(), res.error().message()); return false; } TmpDir::Ptr new_copy_dir = local_context_.storage_config->CreateTypedTmpDir("blob-copy"); if (new_copy_dir == nullptr) { logger_.Emit(LogLevel::Warning, "Failed to create a temporary directory for a blob copy"); return false; } auto new_copy = new_copy_dir->GetPath() / kCopyFileName; if (not FileSystemManager::CopyFile( *blob_path, new_copy, IsExecutableObject(info.type))) { logger_.Emit(LogLevel::Warning, "Failed to create a copy of {}", info.ToString()); return false; } if (not FileSystemManager::CreateFileHardlinkAs( new_copy, target_path, info.type)) { return false; } try { copies->insert_or_assign(info, new_copy_dir); } catch (std::exception const& e) { logger_.Emit(LogLevel::Warning, "Failed to update temp-copies map (continuing anyway): {}", e.what()); } return true; } auto LocalAction::StageInputs( std::filesystem::path const& exec_path, gsl::not_null copies) const noexcept -> bool { if (FileSystemManager::IsRelativePath(exec_path)) { return false; } auto reader = TreeReader{&local_context_.storage->CAS()}; auto result = reader.RecursivelyReadTreeLeafs( root_digest_, exec_path, /*include_trees=*/true); if (not result) { return false; } for (std::size_t i{}; i < result->paths.size(); ++i) { if (not StageInput(result->paths[i], result->infos[i], copies)) { return false; } } return true; } auto LocalAction::CreateDirectoryStructure( std::filesystem::path const& exec_path) const noexcept -> bool { // clean execution directory if (not FileSystemManager::RemoveDirectory(exec_path)) { logger_.Emit(LogLevel::Error, "failed to clean exec_path"); return false; } // create process-exclusive execution directory if (not FileSystemManager::CreateDirectoryExclusive(exec_path)) { logger_.Emit(LogLevel::Error, "failed to exclusively create exec_path"); return false; } // stage inputs (files, leaf trees) to execution directory { LocalAction::FileCopies copies{}; if (not StageInputs(exec_path, &copies)) { logger_.Emit(LogLevel::Error, "failed to stage input files to exec_path"); return false; } } // create output paths auto const create_dir = [this](auto const& dir) { if (not FileSystemManager::CreateDirectory(dir)) { logger_.Emit(LogLevel::Error, "failed to create output directory"); return false; } return true; }; if (mode_ == RequestMode::kV2_1) { return std::all_of( output_paths_.begin(), output_paths_.end(), [this, &exec_path, &create_dir](auto const& local_path) { auto dir = (exec_path / cwd_ / local_path).parent_path(); return create_dir(dir); }); } return std::all_of(output_files_.begin(), output_files_.end(), [this, &exec_path, &create_dir](auto const& local_path) { auto dir = (exec_path / cwd_ / local_path).parent_path(); return create_dir(dir); }) and std::all_of(output_dirs_.begin(), output_dirs_.end(), [this, &exec_path, &create_dir](auto const& local_path) { return create_dir(exec_path / cwd_ / local_path); }); } /// \brief We expect either a regular file, or a symlink. auto LocalAction::CollectOutputFileOrSymlink( std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional { auto file_path = exec_path / local_path; auto type = FileSystemManager::Type(file_path, /*allow_upwards=*/true); if (not type) { logger_.Emit(LogLevel::Error, "expected known type at {}", local_path); return std::nullopt; } if (IsSymlinkObject(*type)) { if (auto content = FileSystemManager::ReadSymlink(file_path)) { // in native mode: check validity of symlink if (ProtocolTraits::IsNative( local_context_.storage->GetHashFunction().GetType()) and not PathIsNonUpwards(*content)) { logger_.Emit( LogLevel::Error, "found invalid symlink at {}", local_path); return std::nullopt; } if (local_context_.storage->CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); return out_symlink; } } } else if (IsFileObject(*type)) { bool is_executable = IsExecutableObject(*type); auto digest = local_context_.storage->CAS().StoreBlob( file_path, is_executable); if (digest) { auto out_file = bazel_re::OutputFile{}; out_file.set_path(local_path); *out_file.mutable_digest() = ArtifactDigestFactory::ToBazel(*digest); out_file.set_is_executable(is_executable); return out_file; } } else { logger_.Emit( LogLevel::Error, "expected file or symlink at {}", local_path); } return std::nullopt; } auto LocalAction::CollectOutputDirOrSymlink( std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional { auto dir_path = exec_path / local_path; auto type = FileSystemManager::Type(dir_path, /*allow_upwards=*/true); if (not type) { logger_.Emit(LogLevel::Error, "expected known type at {}", local_path); return std::nullopt; } if (IsSymlinkObject(*type)) { if (auto content = FileSystemManager::ReadSymlink(dir_path)) { // in native mode: check validity of symlink if (ProtocolTraits::IsNative( local_context_.storage->GetHashFunction().GetType()) and not PathIsNonUpwards(*content)) { logger_.Emit( LogLevel::Error, "found invalid symlink at {}", local_path); return std::nullopt; } if (local_context_.storage->CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); return out_symlink; } } } else if (IsTreeObject(*type)) { if (auto digest = CreateDigestFromLocalOwnedTree( *local_context_.storage, dir_path)) { auto out_dir = bazel_re::OutputDirectory{}; out_dir.set_path(local_path); (*out_dir.mutable_tree_digest()) = ArtifactDigestFactory::ToBazel(*digest); return out_dir; } logger_.Emit(LogLevel::Error, "found invalid entries in directory at {}", local_path); } else { logger_.Emit( LogLevel::Error, "expected directory or symlink at {}", local_path); } return std::nullopt; } auto LocalAction::CollectOutputPath( std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional { auto out_path = exec_path / local_path; auto type = FileSystemManager::Type(out_path, /*allow_upwards=*/true); if (not type) { logger_.Emit(LogLevel::Error, "expected known type at {}", local_path); return std::nullopt; } if (IsSymlinkObject(*type)) { if (auto content = FileSystemManager::ReadSymlink(out_path)) { // in native mode: check validity of symlink if (ProtocolTraits::IsNative( local_context_.storage->GetHashFunction().GetType()) and not PathIsNonUpwards(*content)) { logger_.Emit( LogLevel::Error, "found invalid symlink at {}", local_path); return std::nullopt; } if (local_context_.storage->CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); return out_symlink; } } } else if (IsFileObject(*type)) { bool is_executable = IsExecutableObject(*type); auto digest = local_context_.storage->CAS().StoreBlob( out_path, is_executable); if (digest) { auto out_file = bazel_re::OutputFile{}; out_file.set_path(local_path); *out_file.mutable_digest() = ArtifactDigestFactory::ToBazel(*digest); out_file.set_is_executable(is_executable); return out_file; } } else if (IsTreeObject(*type)) { if (auto digest = CreateDigestFromLocalOwnedTree( *local_context_.storage, out_path)) { auto out_dir = bazel_re::OutputDirectory{}; out_dir.set_path(local_path); (*out_dir.mutable_tree_digest()) = ArtifactDigestFactory::ToBazel(*digest); return out_dir; } logger_.Emit(LogLevel::Error, "found invalid entries in directory at {}", local_path); } else { logger_.Emit(LogLevel::Error, "expected file, directory, or symlink at {}", local_path); } return std::nullopt; } auto LocalAction::CollectAndStoreOutputs( bazel_re::ActionResult* result, std::filesystem::path const& exec_path) const noexcept -> bool { try { logger_.Emit(LogLevel::Trace, "collecting outputs:"); if (mode_ == RequestMode::kV2_1) { for (auto const& path : output_paths_) { auto out = CollectOutputPath(exec_path, path); if (not out) { logger_.Emit(LogLevel::Error, "could not collect output path {}", path); return false; } if (std::holds_alternative(*out)) { auto out_symlink = std::get(*out); logger_.Emit(LogLevel::Trace, " - symlink {}: {}", path, out_symlink.target()); result->mutable_output_symlinks()->Add( std::move(out_symlink)); } else if (std::holds_alternative(*out)) { auto out_file = std::get(*out); auto const& digest = out_file.digest().hash(); logger_.Emit( LogLevel::Trace, " - file {}: {}", path, digest); result->mutable_output_files()->Add(std::move(out_file)); } else { auto out_dir = std::get(*out); auto const& digest = out_dir.tree_digest().hash(); logger_.Emit( LogLevel::Trace, " - dir {}: {}", path, digest); result->mutable_output_directories()->Add( std::move(out_dir)); } } return true; } // if mode_ is RequestMode::kV2_0 or RequestMode::kBestEffort for (auto const& path : output_files_) { auto out = CollectOutputFileOrSymlink(exec_path, path); if (not out) { logger_.Emit(LogLevel::Error, "could not collect output file or symlink {}", path); return false; } if (std::holds_alternative(*out)) { auto out_symlink = std::get(*out); logger_.Emit(LogLevel::Trace, " - symlink {}: {}", path, out_symlink.target()); if (mode_ == RequestMode::kBestEffort) { *result->mutable_output_symlinks()->Add() = out_symlink; } result->mutable_output_file_symlinks()->Add( std::move(out_symlink)); } else { auto out_file = std::get(*out); auto const& digest = out_file.digest().hash(); logger_.Emit(LogLevel::Trace, " - file {}: {}", path, digest); result->mutable_output_files()->Add(std::move(out_file)); } } for (auto const& path : output_dirs_) { auto out = CollectOutputDirOrSymlink(exec_path, path); if (not out) { logger_.Emit(LogLevel::Error, "could not collect output dir or symlink {}", path); return false; } if (std::holds_alternative(*out)) { auto out_symlink = std::get(*out); logger_.Emit(LogLevel::Trace, " - symlink {}: {}", path, out_symlink.target()); if (mode_ == RequestMode::kBestEffort) { *result->mutable_output_symlinks()->Add() = out_symlink; } result->mutable_output_directory_symlinks()->Add( std::move(out_symlink)); } else { auto out_dir = std::get(*out); auto const& digest = out_dir.tree_digest().hash(); logger_.Emit(LogLevel::Trace, " - dir {}: {}", path, digest); result->mutable_output_directories()->Add(std::move(out_dir)); } } return true; } catch (std::exception const& ex) { // should never happen, but report nonetheless logger_.Emit( LogLevel::Error, "collecting outputs failed:\n{}", ex.what()); return false; } } auto LocalAction::DigestFromOwnedFile(std::filesystem::path const& file_path) const noexcept -> std::optional { return local_context_.storage->CAS().StoreBlob( file_path, /*is_executable=*/false); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_action.hpp000066400000000000000000000213501516554100600314720ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP #include #include #include #include #include #include // IWYU pragma: keep #include #include #include #include #include // std::move #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief Action for local execution. class LocalAction final : public IExecutionAction { friend class LocalApi; enum class RequestMode : std::uint8_t { kV2_0, // RBEv2.0 kV2_1, // >=RBEv2.1 kBestEffort // construct both, let the server pick }; public: struct Output { bazel_re::ActionResult action; bool is_cached{}; double duration{}; }; using OutputFileOrSymlink = std::variant; using OutputDirOrSymlink = std::variant; using OutputPath = std::variant; using FileCopies = std::unordered_map; auto Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr final; void SetCacheFlag(CacheFlag flag) noexcept final { cache_flag_ = flag; } void SetTimeout(std::chrono::milliseconds timeout) noexcept final { timeout_ = timeout; } private: Logger logger_{"LocalExecution"}; LocalContext const& local_context_; ArtifactDigest const root_digest_; std::vector const cmdline_; std::string const cwd_; std::vector output_files_; std::vector output_dirs_; std::vector output_paths_; std::map const env_vars_; std::vector const properties_; std::chrono::milliseconds timeout_{kDefaultTimeout}; CacheFlag cache_flag_{CacheFlag::CacheOutput}; RequestMode mode_{}; explicit LocalAction(gsl::not_null local_context, ArtifactDigest root_digest, std::vector command, std::string cwd, std::vector output_files, std::vector output_dirs, std::map env_vars, std::map const& properties, bool best_effort) noexcept : local_context_{*local_context}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, cwd_{std::move(cwd)}, output_files_{std::move(output_files)}, output_dirs_{std::move(output_dirs)}, env_vars_{std::move(env_vars)}, properties_{BazelMsgFactory::CreateMessageVectorFromMap< bazel_re::Platform_Property>(properties)}, mode_{best_effort ? RequestMode::kBestEffort : RequestMode::kV2_0} { std::sort(output_files_.begin(), output_files_.end()); std::sort(output_dirs_.begin(), output_dirs_.end()); if (best_effort) { output_paths_.reserve(output_files_.size() + output_dirs_.size()); output_paths_.insert(output_paths_.end(), output_files_.begin(), output_files_.end()); output_paths_.insert( output_paths_.end(), output_dirs_.begin(), output_dirs_.end()); std::sort(output_paths_.begin(), output_paths_.end()); } } // Alternative constructor with combined output_paths for files and dirs, as // it is used by RBEv2.1 and above. explicit LocalAction( gsl::not_null local_context, ArtifactDigest root_digest, std::vector command, std::string cwd, std::vector output_paths, std::map env_vars, std::map const& properties) noexcept : local_context_{*local_context}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, cwd_{std::move(cwd)}, output_paths_{std::move(output_paths)}, env_vars_{std::move(env_vars)}, properties_{BazelMsgFactory::CreateMessageVectorFromMap< bazel_re::Platform_Property>(properties)}, mode_{RequestMode::kV2_1} { std::sort(output_paths_.begin(), output_paths_.end()); } [[nodiscard]] auto CreateActionDigest(ArtifactDigest const& exec_dir, bool do_not_cache) -> std::optional { auto const env_vars = BazelMsgFactory::CreateMessageVectorFromMap< bazel_re::Command_EnvironmentVariable>(env_vars_); BazelMsgFactory::ActionDigestRequest request{ .command_line = &cmdline_, .cwd = &cwd_, .output_files = &output_files_, .output_dirs = &output_dirs_, .output_paths = &output_paths_, .env_vars = &env_vars, .properties = &properties_, .exec_dir = &exec_dir, .hash_function = local_context_.storage_config->hash_function, .timeout = timeout_, .skip_action_cache = do_not_cache}; return BazelMsgFactory::CreateActionDigestFromCommandLine(request); } [[nodiscard]] auto Run(ArtifactDigest const& action_id) const noexcept -> std::optional; [[nodiscard]] auto StageInput( std::filesystem::path const& target_path, Artifact::ObjectInfo const& info, gsl::not_null copies) const noexcept -> bool; /// \brief Stage input artifacts and leaf trees to the execution directory. /// Stage artifacts and their parent directory structure from CAS to the /// specified execution directory. The execution directory may no exist. /// \param[in] exec_path Absolute path to the execution directory. /// \returns Success indicator. [[nodiscard]] auto StageInputs( std::filesystem::path const& exec_path, gsl::not_null copies) const noexcept -> bool; [[nodiscard]] auto CreateDirectoryStructure( std::filesystem::path const& exec_path) const noexcept -> bool; [[nodiscard]] auto CollectOutputFileOrSymlink( std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional; [[nodiscard]] auto CollectOutputDirOrSymlink( std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional; [[nodiscard]] auto CollectOutputPath(std::filesystem::path const& exec_path, std::string const& local_path) const noexcept -> std::optional; [[nodiscard]] auto CollectAndStoreOutputs( bazel_re::ActionResult* result, std::filesystem::path const& exec_path) const noexcept -> bool; /// \brief Store file from path in file CAS and return pointer to digest. [[nodiscard]] auto DigestFromOwnedFile( std::filesystem::path const& file_path) const noexcept -> std::optional; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_api.cpp000066400000000000000000000347461516554100600307760ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/local/local_api.hpp" #include #include #include #include #include #include // std::nothrow #include #include // std::move #include #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/stream_dumper.hpp" #include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" #include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" namespace { [[nodiscard]] auto CreateFallbackApi( Storage const& storage, RepositoryConfig const* repo_config) noexcept -> std::optional { if (repo_config == nullptr or not ProtocolTraits::IsNative(storage.GetHashFunction().GetType())) { return std::nullopt; } return GitApi{repo_config}; } } // namespace LocalApi::LocalApi(gsl::not_null const& local_context, RepositoryConfig const* repo_config) noexcept : local_context_{*local_context}, git_api_{CreateFallbackApi(*local_context->storage, repo_config)} {} auto LocalApi::CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, std::map const& properties, bool force_legacy) const noexcept -> IExecutionAction::Ptr { if (ProtocolTraits::IsNative(GetHashType())) { // fall back to legacy for native force_legacy = true; } bool best_effort = not force_legacy; return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_, root_digest, command, cwd, output_files, output_dirs, env_vars, properties, best_effort}}; } auto LocalApi::CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_paths, std::map const& env_vars, std::map const& properties) const noexcept -> IExecutionAction::Ptr { return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_, root_digest, command, cwd, output_paths, env_vars, properties}}; } auto LocalApi::RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* /*alternative*/) const noexcept -> bool { if (artifacts_info.size() != output_paths.size()) { Logger::Log(LogLevel::Error, "different number of digests and output paths."); return false; } auto const reader = TreeReader{&local_context_.storage->CAS()}; for (std::size_t i = 0; i < artifacts_info.size(); ++i) { auto const& info = artifacts_info[i]; if (not reader.StageTo({info}, {output_paths[i]})) { if (not git_api_ or not git_api_->RetrieveToPaths({info}, {output_paths[i]})) { Logger::Log(LogLevel::Error, "staging to output path {} failed.", output_paths[i].string()); return false; } } } return true; } auto LocalApi::RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree, IExecutionApi const* /*alternative*/) const noexcept -> bool { auto dumper = StreamDumper{&local_context_.storage->CAS()}; return CommonRetrieveToFds( artifacts_info, fds, [&dumper, &raw_tree](Artifact::ObjectInfo const& info, gsl::not_null const& out) { return dumper.DumpToStream(info, out, raw_tree); }, [&git_api = git_api_, &raw_tree](Artifact::ObjectInfo const& info, int fd) { return git_api and git_api->IsAvailable(info.digest) and git_api->RetrieveToFds({info}, {fd}, raw_tree); }); } auto LocalApi::RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool { // Return immediately if target CAS is this CAS if (this == &api) { return true; } // Determine missing artifacts in other CAS. std::unordered_set> missing; missing.reserve(artifacts_info.size()); { auto back_map = BackMap::Make( &artifacts_info, [](Artifact::ObjectInfo const& info) { return info.digest; }); if (back_map == nullptr) { Logger::Log(LogLevel::Error, "LocalApi: Failed to create BackMap"); return false; } auto missing_digests = api.GetMissingDigests(back_map->GetKeys()); missing = back_map->GetReferences(missing_digests); } // Collect blobs of missing artifacts from local CAS. Trees are // processed recursively before any blob is uploaded. std::unordered_set container; auto const& cas = local_context_.storage->CAS(); for (auto const& info : missing) { // Recursively process trees. if (IsTreeObject(info->type)) { auto reader = TreeReader{&cas}; auto const& result = reader.ReadDirectTreeEntries( info->digest, std::filesystem::path{}); if (not result or not RetrieveToCas(result->infos, api)) { return false; } } // Determine artifact path. auto path = IsTreeObject(info->type) ? cas.TreePath(info->digest) : cas.BlobPath(info->digest, IsExecutableObject(info->type)); if (not path.has_value()) { return false; } auto blob = ArtifactBlob::FromFile(local_context_.storage_config->hash_function, info->type, *std::move(path)); if (not blob.has_value()) { return false; } container.emplace(*std::move(blob)); } // Upload blobs to remote CAS. return api.Upload(std::move(container), /*skip_find_missing=*/true); } auto LocalApi::RetrieveToMemory(Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional { std::optional location{}; if (IsTreeObject(artifact_info.type)) { location = local_context_.storage->CAS().TreePath(artifact_info.digest); } else { location = local_context_.storage->CAS().BlobPath( artifact_info.digest, IsExecutableObject(artifact_info.type)); } std::optional content = std::nullopt; if (location) { content = FileSystemManager::ReadFile(*location); } if (not content and git_api_) { content = git_api_->RetrieveToMemory(artifact_info); } return content; } auto LocalApi::Upload(std::unordered_set&& blobs, bool /*skip_find_missing*/) const noexcept -> bool { // Blobs could have been received over the network, so a simple failure // could result in lost traffic. Try add all blobs and fail if at least // one is corrupted. std::size_t const valid_count = std::count_if( blobs.begin(), blobs.end(), [&cas = local_context_.storage->CAS()](ArtifactBlob const& blob) { bool const is_tree = blob.GetDigest().IsTree(); std::optional cas_digest; if (auto const path = blob.GetFilePath()) { static constexpr bool kOwner = true; cas_digest = is_tree ? cas.StoreTree(*path) : cas.StoreBlob(*path, blob.IsExecutable()); } else if (auto const content = blob.ReadContent()) { cas_digest = is_tree ? cas.StoreTree(*content) : cas.StoreBlob(*content, blob.IsExecutable()); } return cas_digest and *cas_digest == blob.GetDigest(); }); return valid_count == blobs.size(); } auto LocalApi::UploadTree( std::vector const& artifacts) const noexcept -> std::optional { auto build_root = DirectoryTree::FromNamedArtifacts(artifacts); if (not build_root) { Logger::Log(LogLevel::Debug, "failed to create build root from artifacts."); return std::nullopt; } auto const& cas = local_context_.storage->CAS(); if (ProtocolTraits::IsNative(cas.GetHashFunction().GetType())) { return CommonUploadTreeNative(*this, *build_root); } return CommonUploadTreeCompatible( *this, *build_root, [&cas](std::vector const& digests, gsl::not_null*> const& targets) { targets->reserve(digests.size()); for (auto const& digest : digests) { auto p = cas.BlobPath(digest, /*is_executable=*/false); auto content = FileSystemManager::ReadFile(*p); targets->emplace_back(*content); } }); } auto LocalApi::IsAvailable(ArtifactDigest const& digest) const noexcept -> bool { auto found = static_cast( digest.IsTree() ? local_context_.storage->CAS().TreePath(digest) : local_context_.storage->CAS().BlobPath(digest, false)); if ((not found) and git_api_) { if (git_api_->IsAvailable(digest)) { auto plain_local = LocalApi(&local_context_); auto obj_info = std::vector{}; obj_info.push_back(Artifact::ObjectInfo{ digest, digest.IsTree() ? ObjectType::Tree : ObjectType::File, false}); found = git_api_->RetrieveToCas(obj_info, plain_local); } } return found; } auto LocalApi::GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set { std::unordered_set result; result.reserve(digests.size()); for (auto const& digest : digests) { if (not IsAvailable(digest)) { result.emplace(digest); } } return result; } auto LocalApi::SplitBlob(ArtifactDigest const& blob_digest) const noexcept -> std::optional> { Logger::Log(LogLevel::Debug, "SplitBlob({})", blob_digest.hash()); auto split_result = CASUtils::SplitBlobFastCDC(blob_digest, *local_context_.storage); if (not split_result) { Logger::Log(LogLevel::Error, split_result.error().error_message()); return std::nullopt; } auto const& chunk_digests = *split_result; Logger::Log(LogLevel::Debug, [&blob_digest, &chunk_digests]() { std::stringstream ss{}; ss << "Split blob " << blob_digest.hash() << ":" << blob_digest.size() << " into " << chunk_digests.size() << " chunks: [ "; for (auto const& chunk_digest : chunk_digests) { ss << chunk_digest.hash() << ":" << chunk_digest.size() << " "; } ss << "]"; return ss.str(); }); return *std::move(split_result); } auto LocalApi::SpliceBlob(ArtifactDigest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional { Logger::Log(LogLevel::Debug, "SpliceBlob({}, {} chunks)", blob_digest.hash(), chunk_digests.size()); auto splice_result = CASUtils::SpliceBlob( blob_digest, chunk_digests, *local_context_.storage); if (not splice_result) { Logger::Log(LogLevel::Error, splice_result.error().error_message()); return std::nullopt; } return *std::move(splice_result); } auto LocalApi::GetHashType() const noexcept -> HashFunction::Type { return local_context_.storage_config->hash_function.GetType(); } auto LocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr { return local_context_.storage_config->CreateTypedTmpDir("api_space"); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_api.hpp000066400000000000000000000126321516554100600307710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/utils/cpp/tmp_dir.hpp" class LocalApi final : public IExecutionApi { public: explicit LocalApi(gsl::not_null const& local_context, RepositoryConfig const* repo_config = nullptr) noexcept; [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, std::map const& properties, bool force_legacy) const noexcept -> IExecutionAction::Ptr final; /// \brief Create a new action (>=RBEv2.1). /// \param[in] root_digest Digest of the build root. /// \param[in] command Command as argv vector /// \param[in] cwd Working directory, relative to execution root /// \param[in] output_paths List of output file/dir paths, relative to cwd /// \param[in] env_vars The environment variables to set. /// \param[in] properties Platform properties to set. /// \returns The new action. /// Note that, due to missing file/dir separation, the execution response /// will not report output file symlinks and directory symlinks separately. /// (see \ref LocalResponse::DirectorySymlinks) [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_paths, std::map const& env_vars, std::map const& properties) const noexcept -> IExecutionAction::Ptr; [[nodiscard]] auto RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* /*alternative*/) const noexcept -> bool final; [[nodiscard]] auto RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree, IExecutionApi const* /*alternative*/) const noexcept -> bool final; [[nodiscard]] auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool final; [[nodiscard]] auto RetrieveToMemory( Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional override; [[nodiscard]] auto Upload(std::unordered_set&& blobs, bool /*skip_find_missing*/) const noexcept -> bool final; [[nodiscard]] auto UploadTree( std::vector const& artifacts) const noexcept -> std::optional final; [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool final; [[nodiscard]] auto GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set final; [[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest) const noexcept -> std::optional> final; [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final { return true; } [[nodiscard]] auto SpliceBlob( ArtifactDigest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional final; [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final { return true; } [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; private: LocalContext const& local_context_; std::optional const git_api_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_cas_reader.cpp000066400000000000000000000225161516554100600323050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/incremental_reader.hpp" #include "src/utils/cpp/path.hpp" namespace { [[nodiscard]] auto AssembleTree( bazel_re::Directory root, std::unordered_map directories) -> bazel_re::Tree; } // namespace auto LocalCasReader::ReadDirectory(ArtifactDigest const& digest) const noexcept -> std::optional { if (auto const path = cas_.TreePath(digest)) { if (auto const content = FileSystemManager::ReadFile(*path)) { return BazelMsgFactory::MessageFromString( *content); } } Logger::Log( LogLevel::Error, "Directory {} not found in CAS", digest.hash()); return std::nullopt; } auto LocalCasReader::MakeTree(ArtifactDigest const& root) const noexcept -> std::optional { auto const hash_type = cas_.GetHashFunction().GetType(); try { std::unordered_map directories; std::stack to_check; to_check.push(root); while (not to_check.empty()) { auto current = to_check.top(); to_check.pop(); if (directories.contains(current)) { continue; } auto read_dir = ReadDirectory(current); if (not read_dir) { return std::nullopt; } for (auto const& node : read_dir->directories()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, node.digest()); if (not digest) { return std::nullopt; } to_check.push(*std::move(digest)); } directories.insert_or_assign(std::move(current), *std::move(read_dir)); } auto root_directory = directories.extract(root).mapped(); return AssembleTree(std::move(root_directory), std::move(directories)); } catch (...) { return std::nullopt; } } auto LocalCasReader::ReadGitTree(ArtifactDigest const& digest) const noexcept -> std::optional { if (auto const path = cas_.TreePath(digest)) { if (auto const content = FileSystemManager::ReadFile(*path)) { auto check_symlinks = [&cas = cas_](std::vector const& ids) { for (auto const& id : ids) { auto link_path = cas.BlobPath(id, /*is_executable=*/false); if (not link_path) { return false; } // in the local CAS we store as files auto content = FileSystemManager::ReadFile(*link_path); if (not content or not PathIsNonUpwards(*content)) { return false; } } return true; }; // Git-SHA1 hashing is used for reading from git. HashFunction const hash_function{HashFunction::Type::GitSHA1}; return GitRepo::ReadTreeData(*content, digest.hash(), check_symlinks, /*is_hex_id=*/true); } } Logger::Log(LogLevel::Debug, "Tree {} not found in CAS", digest.hash()); return std::nullopt; } auto LocalCasReader::DumpRawTree(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool { auto path = cas_.TreePath(info.digest); return path ? DumpRaw(*path, dumper) : false; } auto LocalCasReader::DumpBlob(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool { auto path = cas_.BlobPath(info.digest, IsExecutableObject(info.type)); return path ? DumpRaw(*path, dumper) : false; } auto LocalCasReader::StageBlobTo( Artifact::ObjectInfo const& info, std::filesystem::path const& output) const noexcept -> bool { if (not IsBlobObject(info.type)) { return false; } auto const blob_path = cas_.BlobPath(info.digest, IsExecutableObject(info.type)); if (not blob_path) { return false; } return FileSystemManager::CreateDirectory(output.parent_path()) and FileSystemManager::CopyFileAs( *blob_path, output, info.type); } auto LocalCasReader::DumpRaw(std::filesystem::path const& path, DumpCallback const& dumper) noexcept -> bool { constexpr std::size_t kChunkSize{512}; auto to_read = IncrementalReader::FromFile(kChunkSize, path); if (not to_read.has_value()) { return false; } return std::all_of( to_read->begin(), to_read->end(), [&dumper](auto const& chunk) { return chunk.has_value() and std::invoke(dumper, std::string{chunk.value()}); }); } auto LocalCasReader::IsNativeProtocol() const noexcept -> bool { return ProtocolTraits::IsNative(cas_.GetHashFunction().GetType()); } auto LocalCasReader::IsDirectoryValid(ArtifactDigest const& digest) const noexcept -> expected { try { auto const hash_type = cas_.GetHashFunction().GetType(); // read directory if (auto const path = cas_.TreePath(digest)) { if (auto const content = FileSystemManager::ReadFile(*path)) { auto dir = BazelMsgFactory::MessageFromString( *content); // check symlinks for (auto const& l : dir->symlinks()) { if (not PathIsNonUpwards(l.target())) { return false; } } // traverse subdirs for (auto const& d : dir->directories()) { auto subdir_digest = ArtifactDigestFactory::FromBazel(hash_type, d.digest()); if (not subdir_digest) { return unexpected{std::move(subdir_digest).error()}; } auto subdir_result = IsDirectoryValid(*std::move(subdir_digest)); if (not subdir_result) { return unexpected{std::move(subdir_result).error()}; } if (not std::move(subdir_result).value()) { return false; } } return true; } } return unexpected{ fmt::format("Directory {} not found in CAS", digest.hash())}; } catch (std::exception const& ex) { return unexpected{fmt::format( "An error occurred checking validity of bazel directory:\n{}", ex.what())}; } } namespace { [[nodiscard]] auto AssembleTree( bazel_re::Directory root, std::unordered_map directories) -> bazel_re::Tree { using Pair = std::pair; std::vector sorted; sorted.reserve(directories.size()); for (auto& p : directories) { sorted.push_back(&p); } std::sort(sorted.begin(), sorted.end(), [](Pair const* l, Pair const* r) { return l->first.hash() < r->first.hash(); }); ::bazel_re::Tree result{}; (*result.mutable_root()) = std::move(root); result.mutable_children()->Reserve(gsl::narrow(sorted.size())); std::transform(sorted.begin(), sorted.end(), ::pb::back_inserter(result.mutable_children()), [](Pair* p) { return std::move(p->second); }); return result; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_cas_reader.hpp000066400000000000000000000056111516554100600323070ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/utils/cpp/expected.hpp" class LocalCasReader final { public: using DumpCallback = std::function; explicit LocalCasReader( gsl::not_null const*> const& cas) noexcept : cas_(*cas) {} [[nodiscard]] auto ReadDirectory(ArtifactDigest const& digest) const noexcept -> std::optional; [[nodiscard]] auto MakeTree(ArtifactDigest const& root) const noexcept -> std::optional; [[nodiscard]] auto ReadGitTree(ArtifactDigest const& digest) const noexcept -> std::optional; [[nodiscard]] auto DumpRawTree(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool; [[nodiscard]] auto DumpBlob(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool; [[nodiscard]] auto StageBlobTo( Artifact::ObjectInfo const& info, std::filesystem::path const& output) const noexcept -> bool; [[nodiscard]] auto IsNativeProtocol() const noexcept -> bool; /// \brief Check recursively if Directory contains any invalid entries /// (i.e., upwards symlinks). /// \returns True if Directory is ok, false if at least on upwards symlink /// has been found, error message on other failures. [[nodiscard]] auto IsDirectoryValid(ArtifactDigest const& digest) const noexcept -> expected; private: LocalCAS const& cas_; [[nodiscard]] static auto DumpRaw(std::filesystem::path const& path, DumpCallback const& dumper) noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/local/local_response.hpp000066400000000000000000000331501516554100600320540ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" #include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/path.hpp" /// \brief Response of a LocalAction. class LocalResponse final : public IExecutionResponse { friend class LocalAction; public: using DirSymlinks = std::unordered_set; auto Status() const noexcept -> StatusCode final { return StatusCode::Success; // unused } auto HasStdErr() const noexcept -> bool final { return (output_.action.stderr_digest().size_bytes() != 0); } auto HasStdOut() const noexcept -> bool final { return (output_.action.stdout_digest().size_bytes() != 0); } auto StdErr() noexcept -> std::string final { if (auto content = ReadContent(output_.action.stderr_digest())) { return *std::move(content); } Logger::Log(LogLevel::Debug, "reading stderr failed"); return {}; } auto StdOut() noexcept -> std::string final { if (auto content = ReadContent(output_.action.stdout_digest())) { return *std::move(content); } Logger::Log(LogLevel::Debug, "reading stdout failed"); return {}; } auto StdErrDigest() noexcept -> std::optional final { auto digest = ArtifactDigestFactory::FromBazel( storage_.GetHashFunction().GetType(), output_.action.stderr_digest()); if (digest) { return *digest; } return std::nullopt; } auto StdOutDigest() noexcept -> std::optional final { auto digest = ArtifactDigestFactory::FromBazel( storage_.GetHashFunction().GetType(), output_.action.stdout_digest()); if (digest) { return *digest; } return std::nullopt; } auto ExitCode() const noexcept -> int final { return output_.action.exit_code(); } auto IsCached() const noexcept -> bool final { return output_.is_cached; }; auto ExecutionDuration() noexcept -> double final { return output_.duration; } auto ActionDigest() const noexcept -> std::string const& final { return action_id_; } auto Artifacts() noexcept -> expected, std::string> final { if (auto error_msg = Populate()) { return unexpected{*std::move(error_msg)}; } return gsl::not_null( &artifacts_); // explicit type needed for expected } // Note that for newer requests (>=RBEv2.1), the local api does not report // output file and directory symlinks separately. In that case, an error // will be returned. auto DirectorySymlinks() noexcept -> expected, std::string> { if (auto error_msg = Populate()) { return unexpected{*std::move(error_msg)}; } bool has_links_apinew = not output_.action.output_symlinks().empty(); bool has_links_apiold = not output_.action.output_file_symlinks().empty() or not output_.action.output_directory_symlinks().empty(); // If the new symlinks field (RBEv2.1) is populated but none of the old // symlinks fields, then the local api did not populate them. if (has_links_apinew and not has_links_apiold) { return unexpected{std::string{ "Local api does not populate the field " "output_directory_symlinks for >=RBEv2.1 requests."}}; } return gsl::not_null( &dir_symlinks_); // explicit type needed for expected } auto HasUpwardsSymlinks() noexcept -> expected final { if (auto error_msg = Populate()) { return unexpected{*std::move(error_msg)}; } return has_upwards_symlinks_; } private: std::string action_id_; LocalAction::Output output_{}; Storage const& storage_; ArtifactInfos artifacts_; DirSymlinks dir_symlinks_; bool has_upwards_symlinks_ = false; // only tracked in compatible mode bool populated_ = false; explicit LocalResponse( std::string action_id, LocalAction::Output output, gsl::not_null const& storage) noexcept : action_id_{std::move(action_id)}, output_{std::move(output)}, storage_{*storage} {} /// \brief Populates the stored data, once. /// \returns Error message on failure, nullopt on success. [[nodiscard]] auto Populate() noexcept -> std::optional { // Initialized only once lazily if (populated_) { return std::nullopt; } ArtifactInfos artifacts{}; auto const& action_result = output_.action; artifacts.reserve( static_cast(action_result.output_files_size()) + static_cast( std::max(action_result.output_symlinks_size(), action_result.output_file_symlinks_size() + action_result.output_directory_symlinks_size())) + static_cast(action_result.output_directories_size())); DirSymlinks dir_symlinks{}; dir_symlinks.reserve(static_cast( action_result.output_directory_symlinks_size())); auto const hash_type = storage_.GetHashFunction().GetType(); // collect files and store them for (auto const& file : action_result.output_files()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, file.digest()); if (not digest) { return fmt::format( "LocalResponse: failed to create artifact digest for {}", file.path()); } try { artifacts.emplace( file.path(), Artifact::ObjectInfo{.digest = *std::move(digest), .type = file.is_executable() ? ObjectType::Executable : ObjectType::File}); } catch (std::exception const& ex) { return fmt::format( "LocalResponse: unexpected failure gathering digest for " "{}:\n{}", file.path(), ex.what()); } } // collect all symlinks and store them for (auto const& link : action_result.output_symlinks()) { try { // in compatible mode: track upwards symlinks has_upwards_symlinks_ = has_upwards_symlinks_ or (not ProtocolTraits::IsNative(hash_type) and not PathIsNonUpwards(link.target())); artifacts.emplace( link.path(), Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs( storage_.GetHashFunction(), link.target()), .type = ObjectType::Symlink}); } catch (std::exception const& ex) { return fmt::format( "LocalResponse: unexpected failure gathering digest for " "{}:\n{}", link.path(), ex.what()); } } for (auto const& link : action_result.output_file_symlinks()) { // DEPRECATED as of v2.1 try { // in compatible mode: track upwards symlinks has_upwards_symlinks_ = has_upwards_symlinks_ or (not ProtocolTraits::IsNative(hash_type) and not PathIsNonUpwards(link.target())); artifacts.emplace( link.path(), Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs( storage_.GetHashFunction(), link.target()), .type = ObjectType::Symlink}); } catch (std::exception const& ex) { return fmt::format( "LocalResponse: unexpected failure gathering digest for " "{}:\n{}", link.path(), ex.what()); } } for (auto const& link : action_result.output_directory_symlinks()) { // DEPRECATED as of v2.1 try { // in compatible mode: track upwards symlinks has_upwards_symlinks_ = has_upwards_symlinks_ or (not ProtocolTraits::IsNative(hash_type) and not PathIsNonUpwards(link.target())); artifacts.emplace( link.path(), Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs( storage_.GetHashFunction(), link.target()), .type = ObjectType::Symlink}); dir_symlinks.emplace(link.path()); // add it to set } catch (std::exception const& ex) { return fmt::format( "LocalResponse: unexpected failure gathering digest for " "{}:\n{}", link.path(), ex.what()); } } // collect directories and store them for (auto const& dir : action_result.output_directories()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, dir.tree_digest()); if (not digest) { return fmt::format( "LocalResponse: failed to create artifact digest for {}", dir.path()); } try { // in compatible mode: track upwards symlinks; requires one // directory traversal; other sources of errors should cause a // fail too, so it is ok to report all traversal errors as // if an invalid entry was found if (not has_upwards_symlinks_ and not ProtocolTraits::IsNative(hash_type)) { LocalCasReader reader{&storage_.CAS()}; auto valid_dir = reader.IsDirectoryValid(*digest); if (not valid_dir) { return std::move(valid_dir).error(); } has_upwards_symlinks_ = not *valid_dir; } artifacts.emplace( dir.path(), Artifact::ObjectInfo{.digest = *std::move(digest), .type = ObjectType::Tree}); } catch (std::exception const& ex) { return fmt::format( "LocalResponse: unexpected failure gathering digest for " "{}:\n{}", dir.path(), ex.what()); } } artifacts_ = std::move(artifacts); dir_symlinks_ = std::move(dir_symlinks); populated_ = true; return std::nullopt; } [[nodiscard]] auto ReadContent(bazel_re::Digest const& digest) const noexcept -> std::optional { auto const a_digest = ArtifactDigestFactory::FromBazel( storage_.GetHashFunction().GetType(), digest); if (not a_digest) { return std::nullopt; } auto const path = storage_.CAS().BlobPath(*a_digest, /*is_executable=*/false); if (not path) { return std::nullopt; } return FileSystemManager::ReadFile(*path); } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/000077500000000000000000000000001516554100600265325ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/TARGETS000066400000000000000000000115211516554100600275660ustar00rootroot00000000000000{ "bazel_network": { "type": ["@", "rules", "CC", "library"] , "name": ["bazel_network"] , "hdrs": [ "bazel/bytestream_client.hpp" , "bazel/bazel_capabilities_client.hpp" , "bazel/bazel_network.hpp" , "bazel/bazel_ac_client.hpp" , "bazel/bazel_cas_client.hpp" , "bazel/bazel_execution_client.hpp" , "bazel/bazel_network_reader.hpp" ] , "srcs": [ "bazel/bazel_capabilities_client.cpp" , "bazel/bazel_network.cpp" , "bazel/bazel_ac_client.cpp" , "bazel/bazel_cas_client.cpp" , "bazel/bazel_execution_client.cpp" , "bazel/bazel_network_reader.cpp" ] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/common/remote", "port"] , ["src/buildtool/common/remote", "retry_config"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "bytestream_utils"] , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/execution_api/common", "message_limits"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] , ["src/utils/cpp", "tmp_dir"] ] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "bazel_remote_apis", "", "semver_proto"] , ["@", "googleapis", "", "google_bytestream_proto"] , ["@", "googleapis", "", "google_longrunning_operations_proto"] , ["@", "googleapis", "", "google_rpc_status_proto"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common/remote", "retry"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "path"] ] } , "bazel_api": { "type": ["@", "rules", "CC", "library"] , "name": ["bazel_api"] , "hdrs": [ "bazel/bazel_api.hpp" , "bazel/bazel_action.hpp" , "bazel/bazel_response.hpp" ] , "srcs": [ "bazel/bazel_api.cpp" , "bazel/bazel_action.cpp" , "bazel/bazel_response.cpp" ] , "deps": [ "bazel_network" , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "port"] , ["src/buildtool/common/remote", "retry_config"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/utils", "outputscheck"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "path"] ] } , "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "hdrs": ["config.hpp"] , "srcs": ["config.cpp"] , "deps": [ ["src/buildtool/common/remote", "remote_common"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/file_system", "file_system_manager"] ] } , "context": { "type": ["@", "rules", "CC", "library"] , "name": ["context"] , "hdrs": ["context.hpp"] , "deps": [ "config" , ["@", "gsl", "", "gsl"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common/remote", "retry_config"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/000077500000000000000000000000001516554100600276275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_ac_client.cpp000066400000000000000000000051641516554100600334370ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include #include #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/retry.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/logging/log_level.hpp" BazelAcClient::BazelAcClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept : retry_config_{*retry_config} { stub_ = bazel_re::ActionCache::NewStub( CreateChannelWithCredentials(server, port, auth)); } auto BazelAcClient::GetActionResult( std::string const& instance_name, bazel_re::Digest const& action_digest, bool inline_stdout, bool inline_stderr, std::vector const& inline_output_files) noexcept -> std::optional { bazel_re::GetActionResultRequest request{}; request.set_instance_name(instance_name); (*request.mutable_action_digest()) = action_digest; request.set_inline_stdout(inline_stdout); request.set_inline_stderr(inline_stderr); std::copy(inline_output_files.begin(), inline_output_files.end(), pb::back_inserter(request.mutable_inline_output_files())); bazel_re::ActionResult response; auto [ok, status] = WithRetry( [this, &response, &request]() { grpc::ClientContext context; return stub_->GetActionResult(&context, request, &response); }, retry_config_, logger_); if (not ok) { if (status.error_code() == grpc::StatusCode::NOT_FOUND) { logger_.Emit( LogLevel::Trace, "cache miss '{}'", status.error_message()); } else { LogStatus(&logger_, LogLevel::Error, status); } return std::nullopt; } return response; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp000066400000000000000000000041631516554100600334420ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP #include #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/logging/logger.hpp" /// Implements client side for service defined here: /// https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L144 class BazelAcClient { public: explicit BazelAcClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept; [[nodiscard]] auto GetActionResult( std::string const& instance_name, bazel_re::Digest const& action_digest, bool inline_stdout, bool inline_stderr, std::vector const& inline_output_files) noexcept -> std::optional; private: RetryConfig const& retry_config_; std::unique_ptr stub_; Logger logger_{"RemoteAcClient"}; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_action.cpp000066400000000000000000000164361516554100600327770ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp" #include #include #include #include // std::move #include "gsl/gsl" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp" #include "src/buildtool/execution_api/utils/outputscheck.hpp" #include "src/buildtool/logging/log_level.hpp" BazelAction::BazelAction(std::shared_ptr network, ArtifactDigest root_digest, std::vector command, std::string cwd, std::vector output_files, std::vector output_dirs, std::map const& env_vars, std::map const& properties, bool best_effort) noexcept : network_{std::move(network)}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, cwd_{std::move(cwd)}, output_files_{std::move(output_files)}, output_dirs_{std::move(output_dirs)}, env_vars_{BazelMsgFactory::CreateMessageVectorFromMap< bazel_re::Command_EnvironmentVariable>(env_vars)}, properties_{BazelMsgFactory::CreateMessageVectorFromMap< bazel_re::Platform_Property>(properties)}, mode_{best_effort ? RequestMode::kBestEffort : RequestMode::kV2_0} { std::sort(output_files_.begin(), output_files_.end()); std::sort(output_dirs_.begin(), output_dirs_.end()); if (best_effort) { output_paths_.reserve(output_files_.size() + output_dirs_.size()); output_paths_.insert( output_paths_.end(), output_files_.begin(), output_files_.end()); output_paths_.insert( output_paths_.end(), output_dirs_.begin(), output_dirs_.end()); std::sort(output_paths_.begin(), output_paths_.end()); } } auto BazelAction::Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr { std::unordered_set blobs{}; auto do_cache = CacheEnabled(cache_flag_); auto action = CreateBundlesForAction(&blobs, root_digest_, not do_cache); if (not action) { if (logger != nullptr) { logger->Emit(LogLevel::Error, "failed to create an action digest for {}", root_digest_.hash()); } return nullptr; } if (logger != nullptr) { logger->Emit(LogLevel::Trace, "start execution\n" " - exec_dir digest: {}\n" " - action digest: {}", root_digest_.hash(), action->hash()); } auto create_response = [](Logger const* logger, std::string const& action_hash, auto&&... args) -> IExecutionResponse::Ptr { try { return IExecutionResponse::Ptr{new BazelResponse{ action_hash, std::forward(args)...}}; } catch (...) { if (logger != nullptr) { logger->Emit(LogLevel::Error, "failed to create a response for {}", action_hash); } } return nullptr; }; if (do_cache) { if (auto result = network_->GetCachedActionResult( *action, mode_ == RequestMode::kV2_0 ? output_files_ : output_paths_)) { if (result->exit_code() == 0 and (mode_ == RequestMode::kV2_0 ? ActionResultContainsExpectedOutputs( *result, output_files_, output_dirs_) : ActionResultContainsExpectedOutputs(*result, output_paths_))) { return create_response( logger, action->hash(), network_, BazelExecutionClient::ExecutionOutput{ .action_result = *result, .cached_result = true}); } } } if (ExecutionEnabled(cache_flag_) and network_->UploadBlobs(std::move(blobs))) { if (auto output = network_->ExecuteBazelActionSync(*action)) { if (cache_flag_ == CacheFlag::PretendCached) { // ensure the same id is created as if caching were enabled auto action_cached = CreateBundlesForAction(nullptr, root_digest_, false); if (not action_cached) { if (logger != nullptr) { logger->Emit( LogLevel::Error, "failed to create a cached action digest for {}", root_digest_.hash()); } return nullptr; } output->cached_result = true; return create_response(logger, action_cached->hash(), network_, *std::move(output)); } return create_response( logger, action->hash(), network_, *std::move(output)); } } return nullptr; } auto BazelAction::CreateBundlesForAction( std::unordered_set* blobs, ArtifactDigest const& exec_dir, bool do_not_cache) const noexcept -> std::optional { using StoreFunc = BazelMsgFactory::ActionDigestRequest::BlobStoreFunc; std::optional store_blob = std::nullopt; if (blobs != nullptr) { store_blob = [&blobs](ArtifactBlob&& blob) { blobs->emplace(std::move(blob)); }; } BazelMsgFactory::ActionDigestRequest request{ .command_line = &cmdline_, .cwd = &cwd_, .output_files = &output_files_, .output_dirs = &output_dirs_, .output_paths = &output_paths_, .env_vars = &env_vars_, .properties = &properties_, .exec_dir = &exec_dir, .hash_function = network_->GetHashFunction(), .timeout = timeout_, .skip_action_cache = do_not_cache, .store_blob = std::move(store_blob)}; auto const action_digest = BazelMsgFactory::CreateActionDigestFromCommandLine(request); if (not action_digest) { return std::nullopt; } return ArtifactDigestFactory::ToBazel(*action_digest); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_action.hpp000066400000000000000000000065111516554100600327750ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP #include #include #include #include #include #include #include #include #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" #include "src/buildtool/logging/logger.hpp" /// \brief Bazel implementation of the abstract Execution Action. /// Uploads all dependencies, creates a Bazel Action and executes it. class BazelAction final : public IExecutionAction { friend class BazelApi; enum class RequestMode : std::uint8_t { kV2_0, // RBEv2.0 kBestEffort // RBEv2.0 and >=RBEv2.1, let the server pick }; public: auto Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr final; void SetCacheFlag(CacheFlag flag) noexcept final { cache_flag_ = flag; } void SetTimeout(std::chrono::milliseconds timeout) noexcept final { timeout_ = timeout; } private: std::shared_ptr const network_; ArtifactDigest const root_digest_; std::vector const cmdline_; std::string const cwd_; std::vector output_files_; std::vector output_dirs_; std::vector output_paths_; std::vector const env_vars_; std::vector const properties_; CacheFlag cache_flag_{CacheFlag::CacheOutput}; std::chrono::milliseconds timeout_{kDefaultTimeout}; RequestMode mode_{}; explicit BazelAction(std::shared_ptr network, ArtifactDigest root_digest, std::vector command, std::string cwd, std::vector output_files, std::vector output_dirs, std::map const& env_vars, std::map const& properties, bool best_effort) noexcept; [[nodiscard]] auto CreateBundlesForAction( std::unordered_set* blobs, ArtifactDigest const& exec_dir, bool do_not_cache) const noexcept -> std::optional; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_api.cpp000066400000000000000000000550501516554100600322660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include #include #include #include #include #include #include #include #include #include #include // std::move #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/stream_dumper.hpp" #include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" namespace { auto constexpr kVersion2dot1 = Capabilities::Version{.major = 2, .minor = 1, .patch = 0}; [[nodiscard]] auto RetrieveToCas( std::unordered_set const& infos, IExecutionApi const& api, std::shared_ptr const& network) noexcept -> bool { auto const back_map = BackMap::Make( &infos, [](Artifact::ObjectInfo const& info) { return info.digest; }); if (back_map == nullptr) { return false; } // Fetch blobs from this CAS: auto reader = network->CreateReader(); auto read_blobs = reader.Read(back_map->GetKeys()); // Restore executable permissions: std::unordered_set result; result.reserve(read_blobs.size()); for (auto it = read_blobs.begin(); it != read_blobs.end();) { auto const info = back_map->GetReference(it->GetDigest()); auto node = read_blobs.extract(it++); node.value().SetExecutable(info.has_value() and IsExecutableObject(info.value()->type)); result.insert(std::move(node)); } auto const all_fetched = result.size() == infos.size(); if (not all_fetched) { Logger::Log(LogLevel::Debug, "could not retrieve all requested blobs."); } // Upload blobs to other CAS. return api.Upload(std::move(result), /*skip_find_missing=*/true) and all_fetched; } [[nodiscard]] auto RetrieveToCasSplitted( Artifact::ObjectInfo const& artifact_info, IExecutionApi const& this_api, IExecutionApi const& other_api, std::shared_ptr const& network) noexcept -> bool { // Split blob into chunks at the remote side and retrieve chunk digests. auto chunk_digests = this_api.SplitBlob(artifact_info.digest); if (not chunk_digests) { // If blob splitting failed, fall back to regular fetching. return ::RetrieveToCas({artifact_info}, other_api, network); } // Fetch unknown chunks. std::unordered_set missing; missing.reserve(chunk_digests->size()); for (auto const& digest : other_api.GetMissingDigests(std::unordered_set( chunk_digests->begin(), chunk_digests->end()))) { missing.emplace( Artifact::ObjectInfo{digest, ObjectType::File, // Chunks are always files /*failed=*/false}); } if (not ::RetrieveToCas(missing, other_api, network)) { return false; } // Assemble blob from chunks. auto digest = other_api.SpliceBlob(artifact_info.digest, *chunk_digests); if (not digest) { // If blob splicing failed, fall back to regular fetching. return ::RetrieveToCas({artifact_info}, other_api, network); } return true; } } // namespace BazelApi::BazelApi(std::string const& instance_name, std::string const& host, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, ExecutionConfiguration const& exec_config, HashFunction hash_function, TmpDir::Ptr temp_space) noexcept { network_ = std::make_shared(instance_name, host, port, auth, retry_config, exec_config, hash_function, std::move(temp_space)); } // implement move constructor in cpp, where all members are complete types BazelApi::BazelApi(BazelApi&& other) noexcept = default; // implement destructor in cpp, where all members are complete types BazelApi::~BazelApi() = default; auto BazelApi::CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, std::map const& properties, bool force_legacy) const noexcept -> IExecutionAction::Ptr { if (ProtocolTraits::IsNative(GetHashType())) { // fall back to legacy for native force_legacy = true; } bool best_effort = not force_legacy; if (not force_legacy and network_->GetCapabilities()->high_api_version < kVersion2dot1) { best_effort = false; } if (not best_effort and network_->GetCapabilities()->low_api_version >= kVersion2dot1) { Logger::Log(LogLevel::Warning, "Server does not support RBEv2.0, falling back to newer " "API version (best effort)."); best_effort = true; } return std::unique_ptr{new (std::nothrow) BazelAction{network_, root_digest, command, cwd, output_files, output_dirs, env_vars, properties, best_effort}}; } [[nodiscard]] auto BazelApi::RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* alternative) const noexcept -> bool { if (artifacts_info.size() != output_paths.size()) { Logger::Log(LogLevel::Warning, "different number of digests and output paths."); return false; } // Obtain file digests from artifact infos std::vector file_digests{}; std::vector artifact_pos{}; for (std::size_t i{}; i < artifacts_info.size(); ++i) { auto const& info = artifacts_info[i]; if (alternative != nullptr and alternative != this and alternative->IsAvailable(info.digest)) { if (not alternative->RetrieveToPaths({info}, {output_paths[i]})) { return false; } } else { if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively auto reader = TreeReader{network_->CreateReader()}; auto const result = reader.RecursivelyReadTreeLeafs( info.digest, output_paths[i]); if (not result or not RetrieveToPaths( result->infos, result->paths, alternative)) { return false; } } else { file_digests.emplace_back(info.digest); artifact_pos.emplace_back(i); } } } // Request file blobs auto const blobs = network_->CreateReader().ReadOrdered(file_digests); if (blobs.size() != file_digests.size()) { Logger::Log(LogLevel::Warning, "could not retrieve all requested blobs."); return false; } for (std::size_t i = 0; i < blobs.size(); ++i) { auto const gpos = artifact_pos[i]; auto const type = artifacts_info[gpos].type; auto const& dst = output_paths[gpos]; bool written = false; if (auto const path = blobs[i].GetFilePath()) { if (FileSystemManager::CreateDirectory(dst.parent_path()) and FileSystemManager::RemoveFile(dst)) { written = IsSymlinkObject(type) ? FileSystemManager::CopySymlinkAs< /*kSetEpochTime=*/true>(*path, dst) : FileSystemManager::CreateFileHardlinkAs< /*kSetEpochTime=*/true>(*path, dst, type); } } if (not written) { Logger::Log(LogLevel::Warning, "staging to output path {} failed.", dst.string()); return false; } } return true; } [[nodiscard]] auto BazelApi::RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree, IExecutionApi const* alternative) const noexcept -> bool { if (alternative == nullptr or alternative == this) { auto dumper = StreamDumper{network_->CreateReader()}; return CommonRetrieveToFds( artifacts_info, fds, [&dumper, &raw_tree](Artifact::ObjectInfo const& info, gsl::not_null const& out) { return dumper.DumpToStream(info, out, raw_tree); }, std::nullopt // no fallback ); } // We have an alternative, and, in fact, preferred API. So go // through the artifacts one by one and first try the the preferred one, // then fall back to retrieving ourselves. if (artifacts_info.size() != fds.size()) { Logger::Log(LogLevel::Error, "different number of digests and file descriptors."); return false; } for (std::size_t i{}; i < artifacts_info.size(); ++i) { auto fd = fds[i]; auto const& info = artifacts_info[i]; if (alternative->IsAvailable(info.digest)) { if (not alternative->RetrieveToFds( std::vector{info}, std::vector{fd}, raw_tree, nullptr)) { return false; } } else { if (not RetrieveToFds(std::vector{info}, std::vector{fd}, raw_tree, nullptr)) { return false; } } } return true; } [[nodiscard]] auto BazelApi::RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool { // Return immediately if target CAS is this CAS if (this == &api) { return true; } // Determine missing artifacts in other CAS. std::unordered_set missing; missing.reserve(artifacts_info.size()); { auto back_map = BackMap::Make( &artifacts_info, [](Artifact::ObjectInfo const& info) { return info.digest; }); if (back_map == nullptr) { Logger::Log(LogLevel::Error, "BazelApi: Failed to create BackMap"); return false; } auto missing_digests = api.GetMissingDigests(back_map->GetKeys()); missing = back_map->GetValues(missing_digests); } // Recursively process trees. auto const reader = TreeReader{network_->CreateReader()}; for (auto const& info : missing) { if (not IsTreeObject(info.type)) { continue; } auto const result = reader.ReadDirectTreeEntries(info.digest, std::filesystem::path{}); if (not result or not RetrieveToCas(result->infos, api)) { return false; } } return ::RetrieveToCas(missing, api, network_); } [[nodiscard]] auto BazelApi::ParallelRetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api, std::size_t jobs, bool use_blob_splitting) const noexcept -> bool { // Return immediately if target CAS is this CAS if (this == &api) { return true; } std::unordered_set done{}; return ParallelRetrieveToCasWithCache( artifacts_info, api, jobs, use_blob_splitting, &done); } [[nodiscard]] auto BazelApi::ParallelRetrieveToCasWithCache( std::vector const& all_artifacts_info, IExecutionApi const& api, std::size_t jobs, bool use_blob_splitting, gsl::not_null*> done) const noexcept -> bool { std::unordered_set artifacts_info; try { artifacts_info.reserve(all_artifacts_info.size()); for (auto const& info : all_artifacts_info) { if (not done->contains(info)) { artifacts_info.emplace(info); } } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "BazelApi: Collecting the set of artifacts failed with:\n{}", ex.what()); return false; } if (artifacts_info.empty()) { return true; // Nothing to do } // Determine missing artifacts in other CAS. std::unordered_set> missing; missing.reserve(artifacts_info.size()); { auto back_map = BackMap::Make( &artifacts_info, [](Artifact::ObjectInfo const& info) { return info.digest; }); if (back_map == nullptr) { Logger::Log(LogLevel::Error, "BazelApi: Failed to create BackMap"); return false; } auto missing_digests = api.GetMissingDigests(back_map->GetKeys()); missing = back_map->GetReferences(missing_digests); } // Recursively process trees. std::atomic_bool failure{false}; std::vector prerequisites{}; std::mutex prerequisites_lock{}; try { auto ts = TaskSystem{jobs}; for (auto const& info : missing) { if (not IsTreeObject(info->type)) { continue; } ts.QueueTask( [this, info, &failure, &prerequisites, &prerequisites_lock]() { auto reader = TreeReader{ network_->CreateReader()}; auto const result = reader.ReadDirectTreeEntries( info->digest, std::filesystem::path{}); if (not result) { failure = true; return; } std::unique_lock lock{prerequisites_lock}; prerequisites.insert(prerequisites.end(), result->infos.begin(), result->infos.end()); }); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Artifact synchronization failed: {}", ex.what()); return false; } if (failure) { return false; } if (not ParallelRetrieveToCasWithCache( prerequisites, api, jobs, use_blob_splitting, done)) { return false; } // In parallel process all the requested artifacts try { auto ts = TaskSystem{jobs}; for (auto const& info : missing) { ts.QueueTask([this, info, &api, &failure, use_blob_splitting]() { if (use_blob_splitting and network_->BlobSplitSupport() and api.BlobSpliceSupport() ? ::RetrieveToCasSplitted(*info, *this, api, network_) : ::RetrieveToCas({*info}, api, network_)) { return; } failure = true; }); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Artifact synchronization failed: {}", ex.what()); return false; } if (failure) { return false; } try { for (auto const& info : artifacts_info) { done->insert(info); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Exception when updating set of synchronized objects " "(continuing anyway): {}", ex.what()); } return true; } [[nodiscard]] auto BazelApi::RetrieveToMemory( Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional { auto reader = network_->CreateReader(); if (auto blob = reader.ReadSingleBlob(artifact_info.digest)) { if (auto const content = blob->ReadContent()) { return *content; } } return std::nullopt; } [[nodiscard]] auto BazelApi::Upload(std::unordered_set&& blobs, bool skip_find_missing) const noexcept -> bool { return network_->UploadBlobs(std::move(blobs), skip_find_missing); } [[nodiscard]] auto BazelApi::UploadTree( std::vector const& artifacts) const noexcept -> std::optional { auto build_root = DirectoryTree::FromNamedArtifacts(artifacts); if (not build_root) { Logger::Log(LogLevel::Debug, "failed to create build root from artifacts."); return std::nullopt; } if (ProtocolTraits::IsNative(network_->GetHashFunction().GetType())) { return CommonUploadTreeNative(*this, *build_root); } return CommonUploadTreeCompatible( *this, *build_root, [&network = network_]( std::vector const& digests, gsl::not_null*> const& targets) { auto reader = network->CreateReader(); targets->reserve(digests.size()); for (auto const& blob : reader.ReadOrdered(digests)) { if (auto const content = blob.ReadContent()) { targets->emplace_back(*content); } } }); } [[nodiscard]] auto BazelApi::IsAvailable( ArtifactDigest const& digest) const noexcept -> bool { return network_->IsAvailable(digest); } [[nodiscard]] auto BazelApi::GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set { return network_->FindMissingBlobs(digests); } [[nodiscard]] auto BazelApi::SplitBlob(ArtifactDigest const& blob_digest) const noexcept -> std::optional> { auto const chunk_digests = network_->SplitBlob(ArtifactDigestFactory::ToBazel(blob_digest)); if (not chunk_digests) { return std::nullopt; } auto artifact_digests = std::vector{}; artifact_digests.reserve(chunk_digests->size()); for (auto const& chunk : *chunk_digests) { auto part = ArtifactDigestFactory::FromBazel( network_->GetHashFunction().GetType(), chunk); if (not part) { return std::nullopt; } artifact_digests.emplace_back(*std::move(part)); } return artifact_digests; } [[nodiscard]] auto BazelApi::BlobSplitSupport() const noexcept -> bool { return network_->BlobSplitSupport(); } [[nodiscard]] auto BazelApi::SpliceBlob( ArtifactDigest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional { auto digests = std::vector{}; digests.reserve(chunk_digests.size()); std::transform(chunk_digests.cbegin(), chunk_digests.cend(), std::back_inserter(digests), [](auto const& artifact_digest) { return ArtifactDigestFactory::ToBazel(artifact_digest); }); auto const digest = network_->SpliceBlob( ArtifactDigestFactory::ToBazel(blob_digest), digests); if (not digest) { return std::nullopt; } auto result = ArtifactDigestFactory::FromBazel( network_->GetHashFunction().GetType(), *digest); if (not result) { return std::nullopt; } return *std::move(result); } [[nodiscard]] auto BazelApi::BlobSpliceSupport() const noexcept -> bool { return network_->BlobSpliceSupport(); } [[nodiscard]] auto BazelApi::GetHashType() const noexcept -> HashFunction::Type { return network_->GetHashFunction().GetType(); } [[nodiscard]] auto BazelApi::GetTempSpace() const noexcept -> TmpDir::Ptr { return network_->GetTempSpace(); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_api.hpp000066400000000000000000000127211516554100600322710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/utils/cpp/tmp_dir.hpp" // forward declaration for actual implementations class BazelNetwork; /// \brief Bazel implementation of the abstract Execution API. class BazelApi final : public IExecutionApi { public: BazelApi(std::string const& instance_name, std::string const& host, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, ExecutionConfiguration const& exec_config, HashFunction hash_function, TmpDir::Ptr temp_space) noexcept; BazelApi(BazelApi const&) = delete; BazelApi(BazelApi&& other) noexcept; auto operator=(BazelApi const&) -> BazelApi& = delete; auto operator=(BazelApi&&) -> BazelApi& = delete; ~BazelApi() final; [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, std::map const& properties, bool force_legacy) const noexcept -> IExecutionAction::Ptr final; [[nodiscard]] auto RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* alternative) const noexcept -> bool final; [[nodiscard]] auto RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, bool raw_tree, IExecutionApi const* alternative) const noexcept -> bool final; [[nodiscard]] auto ParallelRetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api, std::size_t jobs, bool use_blob_splitting) const noexcept -> bool final; [[nodiscard]] auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool final; [[nodiscard]] auto Upload(std::unordered_set&& blobs, bool skip_find_missing) const noexcept -> bool final; [[nodiscard]] auto UploadTree( std::vector const& artifacts) const noexcept -> std::optional final; [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool final; [[nodiscard]] auto GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set final; [[nodiscard]] auto RetrieveToMemory( Artifact::ObjectInfo const& artifact_info) const noexcept -> std::optional final; [[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest) const noexcept -> std::optional> final; [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final; [[nodiscard]] auto SpliceBlob( ArtifactDigest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional final; [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final; [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; private: std::shared_ptr network_; [[nodiscard]] auto ParallelRetrieveToCasWithCache( std::vector const& all_artifacts_info, IExecutionApi const& api, std::size_t jobs, bool use_blob_splitting, gsl::not_null*> done) const noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP bazel_capabilities_client.cpp000066400000000000000000000145411516554100600354250ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include #include #include #include #include #include "build/bazel/semver/semver.pb.h" #include "fmt/core.h" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/retry.hpp" #include "src/buildtool/logging/log_level.hpp" namespace { [[nodiscard]] auto ParseSemVer(build::bazel::semver::SemVer const& version) noexcept -> Capabilities::Version { return Capabilities::Version{.major = version.major(), .minor = version.minor(), .patch = version.patch()}; } [[nodiscard]] auto Parse(std::optional response) noexcept -> Capabilities { if (not response.has_value()) { return Capabilities{}; } // To not duplicate default values here, create default capabilities and // copy data from there. Capabilities const default_capabilities; // If capabilities don't contain cache capabilities, max_batch_total_size is // unlimited (equals 0), or greater than the internal limit, fall back to // the default max_batch_total_size. std::size_t max_batch = default_capabilities.MaxBatchTransferSize; bool split_support = default_capabilities.blob_split_support; bool splice_support = default_capabilities.blob_splice_support; if (response->has_cache_capabilities()) { auto const& cache_capabilities = response->cache_capabilities(); if (cache_capabilities.max_batch_total_size_bytes() != 0) { max_batch = std::min( static_cast(response->cache_capabilities() .max_batch_total_size_bytes()), default_capabilities.MaxBatchTransferSize); } split_support = cache_capabilities.blob_split_support(); splice_support = cache_capabilities.blob_splice_support(); } return Capabilities{ .MaxBatchTransferSize = max_batch, .blob_split_support = split_support, .blob_splice_support = splice_support, .low_api_version = response->has_deprecated_api_version() ? ParseSemVer(response->deprecated_api_version()) : (response->has_low_api_version() // NOLINT ? ParseSemVer(response->low_api_version()) : Capabilities::kMinVersion), .high_api_version = response->has_high_api_version() ? ParseSemVer(response->high_api_version()) : Capabilities::kMaxVersion}; } } // namespace BazelCapabilitiesClient::BazelCapabilitiesClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept : retry_config_{*retry_config} { stub_ = bazel_re::Capabilities::NewStub( CreateChannelWithCredentials(server, port, auth)); } auto BazelCapabilitiesClient::GetCapabilities( std::string const& instance_name) const noexcept -> Capabilities::Ptr { { // Check the cache already contains capabilities for this instance: std::shared_lock guard{lock_}; auto it = capabilities_.find(instance_name); if (it != capabilities_.end()) { return it->second; } } std::optional response; bool is_reasonable_to_retry = true; auto get_capabilities = [&instance_name, &stub = *stub_, &response, &is_reasonable_to_retry]() -> RetryResponse { grpc::ClientContext context; bazel_re::GetCapabilitiesRequest request; *request.mutable_instance_name() = instance_name; bazel_re::ServerCapabilities capabilities; auto const status = stub.GetCapabilities(&context, request, &capabilities); if (status.ok()) { response.emplace(std::move(capabilities)); return RetryResponse{.ok = true}; } is_reasonable_to_retry = IsReasonableToRetry(status); return RetryResponse{ .ok = false, .exit_retry_loop = not is_reasonable_to_retry, .error_msg = fmt::format("While obtaining capabilities: {}", status.error_message())}; }; if (not WithRetry(get_capabilities, retry_config_, logger_, /*fatal_log_level=*/LogLevel::Debug) or not response.has_value()) { logger_.Emit( LogLevel::Warning, "Failed to obtain Capabilities. Falling back to default values."); } bool const cache_result = response.has_value(); auto result = std::make_shared(Parse(std::move(response))); logger_.Emit(LogLevel::Debug, "Obtained server capabilities for \"{}\":\n - " "max_batch_total_size_bytes: {}\n - " "blob_split_support: {}\n - blob_split_support: {}\n", instance_name, result->MaxBatchTransferSize, result->blob_split_support, result->blob_splice_support); // Cache results only if they contain meaningful non-default capabilities or // there's no point in retrying: if (cache_result or not is_reasonable_to_retry) { std::unique_lock lock{lock_}; capabilities_.insert_or_assign(instance_name, result); } return result; } bazel_capabilities_client.hpp000066400000000000000000000064121516554100600354300ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAPABILITIES_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAPABILITIES_CLIENT_HPP #include #include #include #include #include #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/logging/logger.hpp" namespace bazel_re = build::bazel::remote::execution::v2; struct Capabilities final { using Ptr = gsl::not_null>; struct Version { std::int32_t major{}; std::int32_t minor{}; std::int32_t patch{}; [[nodiscard]] auto operator<=>(Version const& other) const noexcept = default; }; static constexpr Version kMinVersion{.major = 0, .minor = 0, .patch = 0}; static constexpr Version kMaxVersion{ .major = std::numeric_limits::max(), .minor = std::numeric_limits::max(), .patch = std::numeric_limits::max()}; std::size_t const MaxBatchTransferSize = MessageLimits::kMaxGrpcLength; bool const blob_split_support = false; bool const blob_splice_support = false; Version const low_api_version = kMinVersion; Version const high_api_version = kMaxVersion; }; class BazelCapabilitiesClient final { public: explicit BazelCapabilitiesClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept; /// \brief Obtain server capabilities for instance_name. /// \return Capabilities corresponding to the given instance_name. Requested /// capabilities are cached if a valid response is received from the server. /// Otherwise, the default capabilities are returned and the caching step is /// skipped to try again next time. [[nodiscard]] auto GetCapabilities( std::string const& instance_name) const noexcept -> Capabilities::Ptr; private: RetryConfig const& retry_config_; std::unique_ptr stub_; Logger logger_{"RemoteCapabilitiesClient"}; mutable std::shared_mutex lock_; mutable std::unordered_map capabilities_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAPABILITIES_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp000066400000000000000000000553441516554100600336270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include #include #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/retry.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" namespace { [[nodiscard]] auto GetContentSize(bazel_re::Digest const& digest) noexcept -> std::size_t { return static_cast(digest.size_bytes()); } [[nodiscard]] auto GetContentSize(ArtifactBlob const& blob) noexcept -> std::size_t { return blob.GetContentSize(); } template [[nodiscard]] auto InitRequest(TRequest* request, TCreator const& request_creator, TIterator begin, TIterator end, std::size_t message_limit, std::optional const& content_limit = std::nullopt) -> TIterator { std::size_t content_size = 0; for (auto it = begin; it != end; ++it) { std::optional to_merge = std::invoke(request_creator, *it); if (not to_merge.has_value()) { return it; } if (request->ByteSizeLong() + to_merge->ByteSizeLong() > message_limit) { return it; } if (content_limit.has_value() and content_size + GetContentSize(*it) > *content_limit) { return it; } request->MergeFrom(*std::move(to_merge)); content_size += GetContentSize(*it); } return end; } } // namespace BazelCasClient::BazelCasClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, gsl::not_null const& capabilities, TmpDir::Ptr temp_space) noexcept : stream_{std::make_unique(server, port, auth)}, retry_config_{*retry_config}, capabilities_{*capabilities}, temp_space_{std::move(temp_space)} { stub_ = bazel_re::ContentAddressableStorage::NewStub( CreateChannelWithCredentials(server, port, auth)); } auto BazelCasClient::BatchReadBlobs( std::string const& instance_name, std::unordered_set const& blobs) const noexcept -> std::unordered_set { std::unordered_set result; if (blobs.empty()) { return result; } auto const max_content_size = GetMaxBatchTransferSize(instance_name); auto const back_map = BackMap::Make( &blobs, ArtifactDigestFactory::ToBazel); if (back_map == nullptr) { return result; } auto request_creator = [&instance_name](bazel_re::Digest const& digest) { bazel_re::BatchReadBlobsRequest request; request.set_instance_name(instance_name); *request.add_digests() = digest; return request; }; try { result.reserve(blobs.size()); bool has_failure = false; for (auto it_processed = back_map->GetKeys().begin(), it = it_processed; it_processed != back_map->GetKeys().end(); it_processed = it) { bazel_re::BatchReadBlobsRequest request; it = InitRequest(&request, request_creator, it, back_map->GetKeys().end(), MessageLimits::kMaxGrpcLength, max_content_size); // If no progress happens, fallback to streaming API: if (it == it_processed) { logger_.Emit(LogLevel::Warning, "BatchReadBlobs: Failed to prepare request for " "{}\nFalling back to streaming API.", it->hash()); std::optional blob; if (auto value = back_map->GetReference(*it)) { blob = ReadSingleBlob(instance_name, *value.value()); } if (blob.has_value()) { result.emplace(*std::move(blob)); } ++it; continue; } logger_.Emit(LogLevel::Trace, "BatchReadBlobs - Request size: {} bytes\n", request.ByteSizeLong()); bool const retry_result = WithRetry( [this, &request, &result, &back_map]() -> RetryResponse { bazel_re::BatchReadBlobsResponse response; grpc::ClientContext context; auto status = stub_->BatchReadBlobs(&context, request, &response); if (status.ok()) { auto batch_response = ProcessBatchResponse< ArtifactBlob, bazel_re::BatchReadBlobsResponse_Response, bazel_re::BatchReadBlobsResponse>( response, [this, &back_map]( std::vector* v, bazel_re::BatchReadBlobsResponse_Response const& r) { auto ref = back_map->GetReference(r.digest()); if (not ref.has_value()) { return; } auto blob = ArtifactBlob::FromTempFile( HashFunction{ref.value()->GetHashType()}, ref.value()->IsTree() ? ObjectType::Tree : ObjectType::File, temp_space_, r.data()); if (not blob.has_value()) { return; } v->emplace_back(*std::move(blob)); }); if (batch_response.ok) { std::move(batch_response.result.begin(), batch_response.result.end(), std::inserter(result, result.end())); return {.ok = true}; } return { .ok = false, .exit_retry_loop = batch_response.exit_retry_loop, .error_msg = batch_response.error_msg}; } auto exit_retry_loop = status.error_code() != grpc::StatusCode::UNAVAILABLE; return { .ok = false, .exit_retry_loop = exit_retry_loop, .error_msg = StatusString(status, "BatchReadBlobs")}; }, retry_config_, logger_); has_failure = has_failure or not retry_result; } if (has_failure) { logger_.Emit(LogLevel::Error, "Failed to BatchReadBlobs."); } } catch (...) { logger_.Emit(LogLevel::Error, "Caught exception in BatchReadBlobs"); } return result; } auto BazelCasClient::UpdateSingleBlob(std::string const& instance_name, ArtifactBlob const& blob) const noexcept -> bool { logger_.Emit(LogLevel::Trace, [&blob]() { std::ostringstream oss{}; oss << "upload single blob" << std::endl; oss << fmt::format(" - {}", blob.GetDigest().hash()) << std::endl; return oss.str(); }); if (not stream_->Write(instance_name, blob)) { logger_.Emit(LogLevel::Error, "Failed to write {}:{}", blob.GetDigest().hash(), blob.GetDigest().size()); return false; } return true; } auto BazelCasClient::IncrementalReadSingleBlob(std::string const& instance_name, ArtifactDigest const& digest) const noexcept -> ByteStreamClient::IncrementalReader { return stream_->IncrementalRead(instance_name, digest); } auto BazelCasClient::ReadSingleBlob(std::string const& instance_name, ArtifactDigest const& digest) const noexcept -> std::optional { return stream_->Read(instance_name, digest, temp_space_); } auto BazelCasClient::SplitBlob(std::string const& instance_name, bazel_re::Digest const& blob_digest) const noexcept -> std::optional> { if (not BlobSplitSupport(instance_name)) { return std::nullopt; } bazel_re::SplitBlobRequest request{}; request.set_instance_name(instance_name); request.mutable_blob_digest()->CopyFrom(blob_digest); bazel_re::SplitBlobResponse response{}; auto [ok, status] = WithRetry( [this, &response, &request]() { grpc::ClientContext context; return stub_->SplitBlob(&context, request, &response); }, retry_config_, logger_); if (not ok) { LogStatus(&logger_, LogLevel::Error, status, "SplitBlob"); return std::nullopt; } std::vector result; result.reserve(response.chunk_digests().size()); std::move(response.mutable_chunk_digests()->begin(), response.mutable_chunk_digests()->end(), std::back_inserter(result)); return result; } auto BazelCasClient::SpliceBlob( std::string const& instance_name, bazel_re::Digest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional { if (not BlobSpliceSupport(instance_name)) { return std::nullopt; } bazel_re::SpliceBlobRequest request{}; request.set_instance_name(instance_name); request.mutable_blob_digest()->CopyFrom(blob_digest); std::copy(chunk_digests.cbegin(), chunk_digests.cend(), pb::back_inserter(request.mutable_chunk_digests())); bazel_re::SpliceBlobResponse response{}; auto [ok, status] = WithRetry( [this, &response, &request]() { grpc::ClientContext context; return stub_->SpliceBlob(&context, request, &response); }, retry_config_, logger_); if (not ok) { LogStatus(&logger_, LogLevel::Error, status, "SpliceBlob"); return std::nullopt; } if (not response.has_blob_digest()) { return std::nullopt; } return response.blob_digest(); } auto BazelCasClient::BlobSplitSupport( std::string const& instance_name) const noexcept -> bool { return capabilities_.GetCapabilities(instance_name)->blob_split_support; } auto BazelCasClient::BlobSpliceSupport( std::string const& instance_name) const noexcept -> bool { return capabilities_.GetCapabilities(instance_name)->blob_splice_support; } auto BazelCasClient::FindMissingBlobs( std::string const& instance_name, std::unordered_set const& digests) const noexcept -> std::unordered_set { std::unordered_set result; if (digests.empty()) { return result; } auto const back_map = BackMap::Make( &digests, ArtifactDigestFactory::ToBazel); if (back_map == nullptr) { return digests; } auto request_creator = [&instance_name](bazel_re::Digest const& digest) { bazel_re::FindMissingBlobsRequest request; request.set_instance_name(instance_name); *request.add_blob_digests() = digest; return request; }; try { result.reserve(digests.size()); for (auto it_processed = back_map->GetKeys().begin(), it = it_processed; it_processed != back_map->GetKeys().end(); it_processed = it) { bazel_re::FindMissingBlobsRequest request; it = InitRequest(&request, request_creator, it, back_map->GetKeys().end(), MessageLimits::kMaxGrpcLength); // If no progress happens, consider current digest missing if (it == it_processed) { logger_.Emit( LogLevel::Warning, "FindMissingBlobs: Failed to prepare request for {}", it->hash()); if (auto value = back_map->GetReference(*it)) { result.emplace(*value.value()); } ++it; continue; } logger_.Emit(LogLevel::Trace, "FindMissingBlobs - Request size: {} bytes\n", request.ByteSizeLong()); bazel_re::FindMissingBlobsResponse response; auto [ok, status] = WithRetry( [this, &response, &request]() { grpc::ClientContext context; return stub_->FindMissingBlobs( &context, request, &response); }, retry_config_, logger_); if (ok) { for (auto const& batch : *response.mutable_missing_blob_digests()) { if (auto value = back_map->GetReference(batch)) { result.emplace(*value.value()); } } } else { LogStatus( &logger_, LogLevel::Error, status, "FindMissingBlobs"); // Failed to get a response. Mark all requested digests missing: for (auto const& digest : request.blob_digests()) { if (auto value = back_map->GetReference(digest)) { result.emplace(*value.value()); } } } } logger_.Emit(LogLevel::Trace, [&digests, &result]() { std::ostringstream oss{}; oss << "find missing blobs" << std::endl; for (auto const& digest : digests) { oss << fmt::format(" - {}", digest.hash()) << std::endl; } oss << "missing blobs" << std::endl; for (auto const& digest : result) { oss << fmt::format(" - {}", digest.hash()) << std::endl; } return oss.str(); }); } catch (...) { logger_.Emit(LogLevel::Error, "Caught exception in FindMissingBlobs"); } return result; } auto BazelCasClient::BatchUpdateBlobs(std::string const& instance_name, std::unordered_set const& blobs) const noexcept -> std::size_t { if (blobs.empty()) { return 0; } auto const max_content_size = GetMaxBatchTransferSize(instance_name); auto request_creator = [&instance_name](ArtifactBlob const& blob) -> std::optional { auto const content = blob.ReadContent(); if (content == nullptr) { return std::nullopt; } bazel_re::BatchUpdateBlobsRequest request; request.set_instance_name(instance_name); auto& r = *request.add_requests(); (*r.mutable_digest()) = ArtifactDigestFactory::ToBazel(blob.GetDigest()); r.set_data(*content); return request; }; std::unordered_set updated; try { updated.reserve(blobs.size()); bool has_failure = false; for (auto it_processed = blobs.begin(), it = it_processed; it_processed != blobs.end(); it_processed = it) { bazel_re::BatchUpdateBlobsRequest request; it = InitRequest(&request, request_creator, it, blobs.end(), MessageLimits::kMaxGrpcLength, max_content_size); // If no progress happens, skip current blob if (it == it_processed) { logger_.Emit( LogLevel::Warning, "BatchUpdateBlobs: Failed to prepare request for {}", it->GetDigest().hash()); ++it; continue; } logger_.Emit(LogLevel::Trace, "BatchUpdateBlobs - Request size: {} bytes\n", request.ByteSizeLong()); bool const retry_result = WithRetry( [this, &request, &updated]() -> RetryResponse { bazel_re::BatchUpdateBlobsResponse response; grpc::ClientContext context; auto status = stub_->BatchUpdateBlobs(&context, request, &response); if (status.ok()) { auto batch_response = ProcessBatchResponse< bazel_re::Digest, bazel_re::BatchUpdateBlobsResponse_Response>( response, [](std::vector* v, bazel_re:: BatchUpdateBlobsResponse_Response const& r) { v->push_back(r.digest()); }); if (batch_response.ok) { std::move(batch_response.result.begin(), batch_response.result.end(), std::inserter(updated, updated.end())); return {.ok = true}; } return { .ok = false, .exit_retry_loop = batch_response.exit_retry_loop, .error_msg = batch_response.error_msg}; } return { .ok = false, .exit_retry_loop = status.error_code() != grpc::StatusCode::UNAVAILABLE, .error_msg = StatusString(status, "BatchUpdateBlobs")}; }, retry_config_, logger_, LogLevel::Performance); has_failure = has_failure or not retry_result; } if (has_failure) { logger_.Emit(LogLevel::Performance, "Failed to BatchUpdateBlobs."); } } catch (...) { logger_.Emit(LogLevel::Warning, "Caught exception in DoBatchUpdateBlobs"); } logger_.Emit(LogLevel::Trace, [&blobs, &updated]() { std::ostringstream oss{}; oss << "upload blobs" << std::endl; for (auto const& blob : blobs) { oss << fmt::format(" - {}", blob.GetDigest().hash()) << std::endl; } oss << "received blobs" << std::endl; for (auto const& digest : updated) { oss << fmt::format(" - {}", digest.hash()) << std::endl; } return oss.str(); }); auto const missing = blobs.size() - updated.size(); if (not updated.empty() and missing != 0) { // The remote execution protocol is a bit unclear about how to deal with // blob updates for which we got no response. While some clients // consider a blob update failed only if a failed response is received, // we are going extra defensive here and also consider missing responses // to be a failed blob update. Issue a retry for the missing blobs. logger_.Emit(LogLevel::Trace, "Retrying with missing blobs"); std::unordered_set missing_blobs; missing_blobs.reserve(missing); for (auto const& blob : blobs) { auto bazel_digest = ArtifactDigestFactory::ToBazel(blob.GetDigest()); if (not updated.contains(bazel_digest)) { missing_blobs.emplace(blob); } } return updated.size() + BatchUpdateBlobs(instance_name, missing_blobs); } if (updated.empty() and missing != 0) { // The batch upload did not make _any_ progress. So there is no value in // trying that again; instead, we fall back to uploading each blob // sequentially. logger_.Emit(LogLevel::Debug, "Falling back to sequential blob upload"); return std::count_if(blobs.begin(), blobs.end(), [this, &instance_name](ArtifactBlob const& blob) { return UpdateSingleBlob(instance_name, blob); }); } return updated.size(); } auto BazelCasClient::GetMaxBatchTransferSize( std::string const& instance_name) const noexcept -> std::size_t { return capabilities_.GetCapabilities(instance_name)->MaxBatchTransferSize; } auto BazelCasClient::CreateGetTreeRequest( std::string const& instance_name, bazel_re::Digest const& root_digest, int page_size, std::string const& page_token) noexcept -> bazel_re::GetTreeRequest { bazel_re::GetTreeRequest request; request.set_instance_name(instance_name); (*request.mutable_root_digest()) = root_digest; request.set_page_size(page_size); request.set_page_token(page_token); return request; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp000066400000000000000000000177611516554100600336350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP #include #include #include #include #include #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bytestream_client.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// Implements client side for serivce defined here: /// https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L317 class BazelCasClient { public: explicit BazelCasClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, gsl::not_null const& capabilities, TmpDir::Ptr temp_space) noexcept; /// \brief Find missing blobs /// \param[in] instance_name Name of the CAS instance /// \param[in] digests The blob digests to search for /// \returns The digests of blobs not found in CAS [[nodiscard]] auto FindMissingBlobs( std::string const& instance_name, std::unordered_set const& digests) const noexcept -> std::unordered_set; /// \brief Upload multiple blobs in batch transfer /// \param[in] instance_name Name of the CAS instance /// \param[in] blobs Blobs to upload /// \returns The count of successfully updated blobs [[nodiscard]] auto BatchUpdateBlobs( std::string const& instance_name, std::unordered_set const& blobs) const noexcept -> std::size_t; /// \brief Read multiple blobs in batch transfer /// \param[in] instance_name Name of the CAS instance /// \param[in] blobs Blob digests to read /// \returns The blobs successfully read [[nodiscard]] auto BatchReadBlobs( std::string const& instance_name, std::unordered_set const& blobs) const noexcept -> std::unordered_set; /// \brief Upload single blob via bytestream /// \param[in] instance_name Name of the CAS instance /// \param[in] blob The blob to upload /// \returns Boolean indicating successful upload [[nodiscard]] auto UpdateSingleBlob(std::string const& instance_name, ArtifactBlob const& blob) const noexcept -> bool; /// \brief Read single blob via incremental bytestream reader /// \param[in] instance_name Name of the CAS instance /// \param[in] digest Blob digest to read /// \returns Incremental bytestream reader. [[nodiscard]] auto IncrementalReadSingleBlob( std::string const& instance_name, ArtifactDigest const& digest) const noexcept -> ByteStreamClient::IncrementalReader; /// \brief Read single blob via bytestream /// \param[in] instance_name Name of the CAS instance /// \param[in] digest Blob digest to read /// \returns The blob successfully read [[nodiscard]] auto ReadSingleBlob(std::string const& instance_name, ArtifactDigest const& digest) const noexcept -> std::optional; /// @brief Split single blob into chunks /// @param[in] instance_name Name of the CAS instance /// @param[in] blob_digest Blob digest to be splitted /// @return The chunk digests of the splitted blob [[nodiscard]] auto SplitBlob(std::string const& instance_name, bazel_re::Digest const& blob_digest) const noexcept -> std::optional>; /// @brief Splice blob from chunks at the remote side /// @param[in] instance_name Name of the CAS instance /// @param[in] blob_digest Expected digest of the spliced blob /// @param[in] chunk_digests The chunk digests of the splitted blob /// @return Whether the splice call was successful [[nodiscard]] auto SpliceBlob( std::string const& instance_name, bazel_re::Digest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional; [[nodiscard]] auto BlobSplitSupport( std::string const& instance_name) const noexcept -> bool; [[nodiscard]] auto BlobSpliceSupport( std::string const& instance_name) const noexcept -> bool; [[nodiscard]] auto GetMaxBatchTransferSize( std::string const& instance_name) const noexcept -> std::size_t; [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr { return temp_space_; } private: std::unique_ptr stream_; RetryConfig const& retry_config_; BazelCapabilitiesClient const& capabilities_; TmpDir::Ptr temp_space_; std::unique_ptr stub_; Logger logger_{"RemoteCasClient"}; [[nodiscard]] static auto CreateGetTreeRequest( std::string const& instance_name, bazel_re::Digest const& root_digest, int page_size, std::string const& page_token) noexcept -> bazel_re::GetTreeRequest; /// \brief Utility class for supporting the Retry strategy while parsing a /// BatchResponse template struct RetryProcessBatchResponse { bool ok{false}; std::vector result{}; bool exit_retry_loop{false}; std::optional error_msg{}; }; // If this function is defined in the .cpp file, clang raises an error // while linking template [[nodiscard]] auto ProcessBatchResponse( TResponse const& response, std::function*, TInner const&)> const& inserter) const noexcept -> RetryProcessBatchResponse { std::vector output; for (auto const& res : response.responses()) { auto const& res_status = res.status(); if (res_status.code() == static_cast(grpc::StatusCode::OK)) { inserter(&output, res); } else { auto exit_retry_loop = (res_status.code() != static_cast(grpc::StatusCode::UNAVAILABLE)); return {.ok = false, .exit_retry_loop = exit_retry_loop, .error_msg = fmt::format("While processing batch response: {}", res_status.ShortDebugString())}; } } return {.ok = true, .result = std::move(output)}; } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP bazel_execution_client.cpp000066400000000000000000000270621516554100600350010ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include // std::move #include #include "fmt/core.h" #include "google/protobuf/any.pb.h" #include "google/protobuf/text_format.h" #include "google/protobuf/timestamp.pb.h" #include "google/rpc/status.pb.h" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/retry.hpp" #include "src/buildtool/logging/log_level.hpp" namespace bazel_re = build::bazel::remote::execution::v2; namespace { void LogExecutionStatus(gsl::not_null const& logger, google::rpc::Status const& s) noexcept { switch (s.code()) { case grpc::StatusCode::DEADLINE_EXCEEDED: logger->Emit(LogLevel::Error, "Execution timed out."); break; case grpc::StatusCode::UNAVAILABLE: // quote from remote_execution.proto: // Due to a transient condition, such as all workers being occupied // (and the server does not support a queue), the action could not // be started. The client should retry. logger->Emit(LogLevel::Debug, "Execution could not be started.\n{}", s.ShortDebugString()); break; case grpc::StatusCode::FAILED_PRECONDITION: // quote from remote_execution.proto: // One or more errors occurred in setting up the // action requested, such as a missing input or command or no worker // being available. The client may be able to fix the errors and // retry. logger->Emit(LogLevel::Progress, "Some precondition for the action failed.\n{}", s.message()); break; default: // fallback to default status logging LogStatus(logger, LogLevel::Error, s); break; } } auto DebugString(grpc::Status const& status) -> std::string { return fmt::format("{}: {}", static_cast(status.error_code()), status.error_message()); } } // namespace BazelExecutionClient::BazelExecutionClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept : retry_config_{*retry_config} { stub_ = bazel_re::Execution::NewStub( CreateChannelWithCredentials(server, port, auth)); } auto BazelExecutionClient::Execute(std::string const& instance_name, bazel_re::Digest const& action_digest, ExecutionConfiguration const& config, bool wait) -> BazelExecutionClient::ExecutionResponse { auto execution_policy = std::make_unique(); execution_policy->set_priority(config.execution_priority); auto results_cache_policy = std::make_unique(); results_cache_policy->set_priority(config.results_cache_priority); bazel_re::ExecuteRequest request; request.set_instance_name(instance_name); request.set_skip_cache_lookup(config.skip_cache_lookup); (*request.mutable_action_digest()) = action_digest; request.set_allocated_execution_policy(execution_policy.release()); request.set_allocated_results_cache_policy(results_cache_policy.release()); BazelExecutionClient::ExecutionResponse response; auto execute = [this, &request, wait, &response]() -> RetryResponse { grpc::ClientContext context; std::unique_ptr> reader(stub_->Execute(&context, request)); auto [op, fatal, _] = ReadExecution(reader.get(), wait); if (not op.has_value()) { return {.ok = false, .exit_retry_loop = fatal}; } auto contents = ExtractContents(std::move(op)); response = contents.response; if (response.state == ExecutionResponse::State::Ongoing) { return {.ok = true, .exit_retry_loop = true}; } if (response.state == ExecutionResponse::State::Finished) { return {.ok = true}; } auto const is_fatal = response.state != ExecutionResponse::State::Retry; return {.ok = false, .exit_retry_loop = is_fatal, .error_msg = is_fatal ? std::nullopt : contents.error_msg}; }; if (not WithRetry(execute, retry_config_, logger_)) { logger_.Emit(LogLevel::Error, "Failed to execute action {}.", action_digest.ShortDebugString()); } return response; } auto BazelExecutionClient::WaitExecution(std::string const& execution_handle) -> BazelExecutionClient::ExecutionResponse { bazel_re::WaitExecutionRequest request; request.set_name(execution_handle); BazelExecutionClient::ExecutionResponse response; auto wait_execution = [this, &request, &response]() -> RetryResponse { grpc::ClientContext context; std::unique_ptr> reader(stub_->WaitExecution(&context, request)); auto [op, fatal, _] = ReadExecution(reader.get(), /*wait=*/true); if (not op.has_value()) { return {.ok = false, .exit_retry_loop = fatal}; } auto contents = ExtractContents(std::move(op)); response = contents.response; if (response.state == ExecutionResponse::State::Finished) { return {.ok = true}; } auto const is_fatal = response.state != ExecutionResponse::State::Retry; return {.ok = false, .exit_retry_loop = is_fatal, .error_msg = is_fatal ? std::nullopt : contents.error_msg}; }; if (not WithRetry(wait_execution, retry_config_, logger_)) { logger_.Emit( LogLevel::Error, "Failed to Execute action {}.", request.name()); } return response; } auto BazelExecutionClient::ReadExecution( grpc::ClientReader* reader, bool wait) -> RetryReadOperation { if (reader == nullptr) { grpc::Status status{grpc::StatusCode::UNKNOWN, "Reader unavailable"}; LogStatus(&logger_, LogLevel::Error, status); return {.operation = std::nullopt, .exit_retry_loop = true, .error_msg = DebugString(status)}; } google::longrunning::Operation operation; if (not reader->Read(&operation)) { grpc::Status status = reader->Finish(); auto exit_retry_loop = (status.error_code() != grpc::StatusCode::UNAVAILABLE) && (status.error_code() != grpc::StatusCode::DEADLINE_EXCEEDED); LogStatus(&logger_, (exit_retry_loop ? LogLevel::Error : LogLevel::Debug), status); return {std::nullopt, exit_retry_loop, DebugString(status)}; } // Important note: do not call reader->Finish() unless reader->Read() // returned false, otherwise the thread will be never released if (wait) { while (reader->Read(&operation)) { } grpc::Status status = reader->Finish(); if (not status.ok()) { auto exit_retry_loop = (status.error_code() != grpc::StatusCode::UNAVAILABLE) && (status.error_code() != grpc::StatusCode::DEADLINE_EXCEEDED); LogStatus(&logger_, (exit_retry_loop ? LogLevel::Error : LogLevel::Debug), status); return {std::nullopt, exit_retry_loop, DebugString(status)}; } } return {.operation = operation, .exit_retry_loop = false}; } auto BazelExecutionClient::ExtractContents( std::optional&& operation) -> RetryExtractContents { if (not operation) { // Error was already logged in ReadExecution() return {ExecutionResponse::MakeEmptyFailed(), std::nullopt}; } ExecutionResponse response; response.execution_handle = operation->name(); if (not operation->done()) { response.state = ExecutionResponse::State::Ongoing; return {response, std::nullopt}; } if (operation->has_error()) { LogStatus(&logger_, LogLevel::Debug, operation->error()); if (operation->error().code() == grpc::StatusCode::UNAVAILABLE) { response.state = ExecutionResponse::State::Retry; } else { response.state = ExecutionResponse::State::Failed; } return {response, operation->error().ShortDebugString()}; } // Get execution response Unpacked from Protobufs Any type to the actual // type in our case auto const& raw_response = operation->response(); if (not raw_response.Is()) { // Fatal error, the type should be correct logger_.Emit(LogLevel::Error, "Corrupted ExecuteResponse"); response.state = ExecutionResponse::State::Failed; return {response, "Corrupted ExecuteResponse"}; } bazel_re::ExecuteResponse exec_response; raw_response.UnpackTo(&exec_response); auto status_code = exec_response.status().code(); if (status_code != grpc::StatusCode::OK) { LogExecutionStatus(&logger_, exec_response.status()); if (status_code == grpc::StatusCode::UNAVAILABLE) { response.state = ExecutionResponse::State::Retry; } else if (status_code == grpc::StatusCode::FAILED_PRECONDITION) { logger_.Emit(LogLevel::Debug, [&exec_response] { std::string text_repr; google::protobuf::TextFormat::PrintToString(exec_response, &text_repr); return fmt::format( "Full exec_response of precondition failure\n{}", text_repr); }); response.state = ExecutionResponse::State::Retry; } else { response.state = ExecutionResponse::State::Failed; } return {response, exec_response.status().ShortDebugString()}; } ExecutionOutput output; output.action_result = exec_response.result(); output.cached_result = exec_response.cached_result(); output.message = exec_response.message(); for (const auto& [k, v] : exec_response.server_logs()) { output.server_logs[k].CopyFrom(v); } auto metadata = exec_response.result().execution_metadata(); auto const& start_time = metadata.worker_start_timestamp(); auto const& stop_time = metadata.worker_completed_timestamp(); const double k_one_nanosecond_in_seconds = 1e-9; output.duration = static_cast(stop_time.seconds() - start_time.seconds()) + k_one_nanosecond_in_seconds * static_cast(stop_time.nanos() - start_time.nanos()); response.output = output; response.state = ExecutionResponse::State::Finished; return {response, std::nullopt}; } bazel_execution_client.hpp000066400000000000000000000076221516554100600350060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP #include #include #include #include #include #include #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "google/longrunning/operations.pb.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/logging/logger.hpp" /// Implements client side for service defined here: /// https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L44 class BazelExecutionClient { public: struct ExecutionOutput { bazel_re::ActionResult action_result; bool cached_result{}; double duration{}; grpc::Status status{}; std::unordered_map server_logs{}; std::string message{}; }; struct ExecutionResponse { enum class State : std::uint8_t { Failed, Ongoing, Finished, Unknown, Retry }; std::string execution_handle; State state{State::Unknown}; std::optional output{std::nullopt}; static auto MakeEmptyFailed() -> ExecutionResponse { return ExecutionResponse{.execution_handle = {}, .state = ExecutionResponse::State::Failed, .output = std::nullopt}; } }; explicit BazelExecutionClient( std::string const& server, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config) noexcept; [[nodiscard]] auto Execute(std::string const& instance_name, bazel_re::Digest const& action_digest, ExecutionConfiguration const& config, bool wait) -> ExecutionResponse; [[nodiscard]] auto WaitExecution(std::string const& execution_handle) -> ExecutionResponse; private: RetryConfig const& retry_config_; std::unique_ptr stub_; Logger logger_{"RemoteExecutionClient"}; struct RetryReadOperation { std::optional operation{std::nullopt}; bool exit_retry_loop{false}; std::optional error_msg{std::nullopt}; }; struct RetryExtractContents { ExecutionResponse response; std::optional error_msg{std::nullopt}; }; [[nodiscard]] auto ReadExecution( grpc::ClientReader* reader, bool wait) -> RetryReadOperation; [[nodiscard]] auto ExtractContents( std::optional&& operation) -> RetryExtractContents; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_network.cpp000066400000000000000000000140441516554100600332040ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" #include #include #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/back_map.hpp" BazelNetwork::BazelNetwork( std::string instance_name, std::string const& host, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, ExecutionConfiguration const& exec_config, HashFunction hash_function, TmpDir::Ptr temp_space) noexcept : instance_name_{std::move(instance_name)}, capabilities_{std::make_unique(host, port, auth, retry_config)}, cas_{std::make_unique(host, port, auth, retry_config, capabilities_.get(), std::move(temp_space))}, ac_{std::make_unique(host, port, auth, retry_config)}, exec_{std::make_unique(host, port, auth, retry_config)}, exec_config_{exec_config}, hash_function_{hash_function} {} auto BazelNetwork::IsAvailable(ArtifactDigest const& digest) const noexcept -> bool { return cas_->FindMissingBlobs(instance_name_, {digest}).empty(); } auto BazelNetwork::FindMissingBlobs( std::unordered_set const& digests) const noexcept -> std::unordered_set { return cas_->FindMissingBlobs(instance_name_, digests); } auto BazelNetwork::SplitBlob(bazel_re::Digest const& blob_digest) const noexcept -> std::optional> { return cas_->SplitBlob(instance_name_, blob_digest); } auto BazelNetwork::SpliceBlob( bazel_re::Digest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional { return cas_->SpliceBlob(instance_name_, blob_digest, chunk_digests); } auto BazelNetwork::BlobSplitSupport() const noexcept -> bool { return cas_->BlobSplitSupport(instance_name_); } auto BazelNetwork::BlobSpliceSupport() const noexcept -> bool { return cas_->BlobSpliceSupport(instance_name_); } auto BazelNetwork::DoUploadBlobs( std::unordered_set blobs) noexcept -> bool { if (blobs.empty()) { return true; } try { // First upload all blobs that must use bytestream api because of their // size: for (auto it = blobs.begin(); it != blobs.end();) { if (it->GetContentSize() <= MessageLimits::kMaxGrpcLength) { ++it; continue; } if (not cas_->UpdateSingleBlob(instance_name_, *it)) { return false; } it = blobs.erase(it); } // After uploading via stream api, only small blobs that may be uploaded // using batch are in the container: return cas_->BatchUpdateBlobs(instance_name_, blobs) == blobs.size(); } catch (...) { Logger::Log(LogLevel::Warning, "Unknown exception"); return false; } } auto BazelNetwork::UploadBlobs(std::unordered_set&& blobs, bool skip_find_missing) noexcept -> bool { if (not skip_find_missing) { auto const back_map = BackMap::Make( &blobs, [](ArtifactBlob const& blob) { return blob.GetDigest(); }); if (back_map == nullptr) { return false; } // back_map becomes invalid after this call: blobs = back_map->GetValues(FindMissingBlobs(back_map->GetKeys())); } return DoUploadBlobs(std::move(blobs)); } auto BazelNetwork::ExecuteBazelActionSync( bazel_re::Digest const& action) noexcept -> std::optional { auto response = exec_->Execute(instance_name_, action, exec_config_, true /*wait*/); if (response.state == BazelExecutionClient::ExecutionResponse::State::Ongoing) { Logger::Log( LogLevel::Trace, "Waiting for {}", response.execution_handle); response = exec_->WaitExecution(response.execution_handle); } if (response.state != BazelExecutionClient::ExecutionResponse::State::Finished or not response.output) { Logger::Log(LogLevel::Warning, "Failed to execute action with execution id {}.", action.hash()); return std::nullopt; } return response.output; } auto BazelNetwork::CreateReader() const noexcept -> BazelNetworkReader { return BazelNetworkReader{instance_name_, cas_.get(), hash_function_}; } auto BazelNetwork::GetCachedActionResult( bazel_re::Digest const& action, std::vector const& output_files) const noexcept -> std::optional { return ac_->GetActionResult( instance_name_, action, false, false, output_files); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_network.hpp000066400000000000000000000115161516554100600332120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief Contains all network clients and is responsible for all network IO. class BazelNetwork { public: explicit BazelNetwork(std::string instance_name, std::string const& host, Port port, gsl::not_null const& auth, gsl::not_null const& retry_config, ExecutionConfiguration const& exec_config, HashFunction hash_function, TmpDir::Ptr temp_space) noexcept; /// \brief Check if digest exists in CAS /// \param[in] digest The digest to look up /// \returns True if digest exists in CAS, false otherwise [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool; [[nodiscard]] auto FindMissingBlobs( std::unordered_set const& digests) const noexcept -> std::unordered_set; [[nodiscard]] auto SplitBlob(bazel_re::Digest const& blob_digest) const noexcept -> std::optional>; [[nodiscard]] auto SpliceBlob( bazel_re::Digest const& blob_digest, std::vector const& chunk_digests) const noexcept -> std::optional; [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool; [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool; /// \brief Uploads blobs to CAS /// \param blobs The blobs to upload /// \param skip_find_missing Skip finding missing blobs, just upload all /// \returns True if upload was successful, false otherwise [[nodiscard]] auto UploadBlobs(std::unordered_set&& blobs, bool skip_find_missing = false) noexcept -> bool; [[nodiscard]] auto ExecuteBazelActionSync( bazel_re::Digest const& action) noexcept -> std::optional; [[nodiscard]] auto CreateReader() const noexcept -> BazelNetworkReader; [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction { return hash_function_; } [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr { return cas_->GetTempSpace(); } [[nodiscard]] auto GetCachedActionResult( bazel_re::Digest const& action, std::vector const& output_files) const noexcept -> std::optional; [[nodiscard]] auto GetCapabilities() const noexcept -> Capabilities::Ptr { return capabilities_->GetCapabilities(instance_name_); } private: std::string const instance_name_; std::unique_ptr capabilities_; std::unique_ptr cas_; std::unique_ptr ac_; std::unique_ptr exec_; ExecutionConfiguration exec_config_{}; HashFunction hash_function_; [[nodiscard]] auto DoUploadBlobs( std::unordered_set blobs) noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp000066400000000000000000000176351516554100600345370ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include #include #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/path.hpp" BazelNetworkReader::BazelNetworkReader( std::string instance_name, gsl::not_null const& cas, HashFunction hash_function) noexcept : instance_name_{std::move(instance_name)}, cas_{*cas}, hash_function_{hash_function} {} auto BazelNetworkReader::ReadDirectory(ArtifactDigest const& digest) const noexcept -> std::optional { auto blob = ReadSingleBlob(digest); if (not blob.has_value()) { Logger::Log( LogLevel::Debug, "BazelNetworkReader::ReadDirectory: Directory {} not found in CAS", digest.hash()); return std::nullopt; } auto const content = blob->ReadContent(); if (content == nullptr) { Logger::Log( LogLevel::Debug, "BazelNetworkReader::ReadDirectory: Failed to read directory {}", digest.hash()); return std::nullopt; } auto dir = BazelMsgFactory::MessageFromString(*content); if (not dir.has_value()) { Logger::Log(LogLevel::Debug, "BazelNetworkReader::ReadDirectory: Failed to parse " "directory content {}", digest.hash()); return std::nullopt; } std::vector symlinks; symlinks.reserve(dir->symlinks().size()); for (const auto& link : dir->symlinks()) { auto blob = ArtifactBlob::FromMemory( hash_function_, ObjectType::File, link.target()); if (blob.has_value()) { symlinks.push_back(*std::move(blob)); } else { Logger::Log(LogLevel::Debug, "BazelNetworkReader::ReadDirectory: Failed to create " "an ArtifactBlob from symlink {} -> {}", link.name(), link.target()); } } auto back_map = BackMap::Make( &symlinks, [](ArtifactBlob const& blob) { return blob.GetDigest(); }); auto missing = cas_.FindMissingBlobs(instance_name_, back_map->GetKeys()); if (not missing.empty() and cas_.BatchUpdateBlobs(instance_name_, back_map->GetValues(missing)) != missing.size()) { Logger::Log( LogLevel::Debug, "BazelNetworkReader::ReadDirectory: Failed to upload all symlinks"); } return dir; } auto BazelNetworkReader::ReadGitTree(ArtifactDigest const& digest) const noexcept -> std::optional { ExpectsAudit(IsNativeProtocol()); auto read_blob = ReadSingleBlob(digest); if (not read_blob) { Logger::Log(LogLevel::Debug, "Tree {} not found in CAS", digest.hash()); return std::nullopt; } auto const content = read_blob->ReadContent(); if (content == nullptr) { return std::nullopt; } auto check_symlinks = [this](std::vector const& ids) { auto const blobs = ReadOrdered(ids); if (blobs.size() != ids.size()) { Logger::Log(LogLevel::Debug, "BazelNetworkReader::ReadGitTree: read wrong number of " "symlinks."); return false; } return std::all_of( blobs.begin(), blobs.end(), [](ArtifactBlob const& blob) { auto const content = blob.ReadContent(); return content != nullptr and PathIsNonUpwards(*content); }); }; return GitRepo::ReadTreeData(*content, digest.hash(), check_symlinks, /*is_hex_id=*/true); } auto BazelNetworkReader::DumpRawTree(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool { auto read_blob = ReadSingleBlob(info.digest); if (not read_blob) { Logger::Log( LogLevel::Debug, "Object {} not found in CAS", info.digest.hash()); return false; } try { auto const content = read_blob->ReadContent(); return content != nullptr and std::invoke(dumper, *content); } catch (...) { return false; } } auto BazelNetworkReader::DumpBlob(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool { auto reader = cas_.IncrementalReadSingleBlob(instance_name_, info.digest); auto data = reader.Next(); while (data and not data->empty()) { try { if (not std::invoke(dumper, *data)) { return false; } } catch (...) { return false; } data = reader.Next(); } return data.has_value(); } auto BazelNetworkReader::IsNativeProtocol() const noexcept -> bool { return ProtocolTraits::IsNative(hash_function_.GetType()); } auto BazelNetworkReader::ReadSingleBlob(ArtifactDigest const& digest) const noexcept -> std::optional { return cas_.ReadSingleBlob(instance_name_, digest); } auto BazelNetworkReader::Read(std::unordered_set const& digests) const noexcept -> std::unordered_set { std::unordered_set read_result; read_result.reserve(digests.size()); std::unordered_set to_batch; to_batch.reserve(digests.size()); // Upload blobs that don't fit for batching: size is larger than limit or // unknown std::size_t const limit = cas_.GetMaxBatchTransferSize(instance_name_); for (auto const& digest : digests) { if (digest.size() == 0 or digest.size() > limit) { auto blob = cas_.ReadSingleBlob(instance_name_, digest); if (blob.has_value()) { read_result.emplace(*std::move(blob)); } } else { to_batch.emplace(digest); } } // Batch remaining blobs: read_result.merge(cas_.BatchReadBlobs(instance_name_, to_batch)); return read_result; } auto BazelNetworkReader::ReadOrdered(std::vector const& digests) const noexcept -> std::vector { auto const read_result = Read(std::unordered_set(digests.begin(), digests.end())); auto const back_map = BackMap::Make( &read_result, [](ArtifactBlob const& blob) { return blob.GetDigest(); }); if (back_map == nullptr) { return {}; } std::vector artifacts; artifacts.reserve(digests.size()); for (auto const& digest : digests) { if (auto value = back_map->GetReference(digest)) { artifacts.emplace_back(*value.value()); } } return artifacts; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp000066400000000000000000000061451516554100600345360ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/file_system/git_repo.hpp" class BazelNetworkReader final { class IncrementalReader; public: using DumpCallback = std::function; explicit BazelNetworkReader(std::string instance_name, gsl::not_null const& cas, HashFunction hash_function) noexcept; [[nodiscard]] auto ReadDirectory(ArtifactDigest const& digest) const noexcept -> std::optional; [[nodiscard]] auto ReadGitTree(ArtifactDigest const& digest) const noexcept -> std::optional; [[nodiscard]] auto DumpRawTree(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool; [[nodiscard]] auto DumpBlob(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool; // NOLINTNEXTLINE(readability-convert-member-functions-to-static) [[nodiscard]] auto StageBlobTo( Artifact::ObjectInfo const& /*info*/, std::filesystem::path const& /*output*/) const noexcept -> bool { return false; // not implemented } [[nodiscard]] auto IsNativeProtocol() const noexcept -> bool; [[nodiscard]] auto ReadSingleBlob(ArtifactDigest const& digest) const noexcept -> std::optional; [[nodiscard]] auto Read(std::unordered_set const& digests) const noexcept -> std::unordered_set; [[nodiscard]] auto ReadOrdered(std::vector const& digests) const noexcept -> std::vector; private: std::string const instance_name_; BazelCasClient const& cas_; HashFunction hash_function_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_response.cpp000066400000000000000000000373241516554100600333570ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp" #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/path.hpp" namespace { /// \brief Generates an ArtifactBlob from a Directory message and while /// there, checks whether it includes any upward symlinks and collects /// all symlinks as separate ArtifactBlobs. /// \param[in] hash_function The hash function to be used. /// \param[in] dir The Bazel Directory message. /// \returns In case of success: a tuple containing the ArtifactBlob of /// the Directory message, the collection of ArtifactBlobs of all /// contained symbolic links, and a flag whether the message contains /// upward symlinks. In case of failure: an error message. auto ProcessDirectoryMessage(HashFunction hash_function, bazel_re::Directory const& dir) noexcept -> expected, /*has_upwards_symlinks*/ bool>, std::string> { // in compatible mode: track upwards symlinks bool has_upwards_symlinks = std::any_of( dir.symlinks().begin(), dir.symlinks().end(), [](auto const& link) { return not PathIsNonUpwards(link.target()); }); auto blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, dir.SerializeAsString()); if (not blob) { return unexpected{std::move(blob).error()}; } // extract and return symlinks embedded in the Bazel Directory // message as separate blobs to upload them later std::unordered_set link_blobs; for (auto const& link : dir.symlinks()) { auto link_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, link.target()); if (not link_blob) { return unexpected{std::move(link_blob).error()}; } link_blobs.insert(std::move(link_blob).value()); } return std::make_tuple( std::move(blob).value(), std::move(link_blobs), has_upwards_symlinks); } } // namespace auto BazelResponse::ReadStringBlob(bazel_re::Digest const& id) noexcept -> std::string { auto digest = ArtifactDigestFactory::FromBazel( network_->GetHashFunction().GetType(), id); if (digest.has_value()) { auto reader = network_->CreateReader(); if (auto blob = reader.ReadSingleBlob(*digest)) { if (auto const content = blob->ReadContent()) { return *content; } } } Logger::Log(LogLevel::Warning, "reading digest {} from action response failed", id.hash()); return std::string{}; } auto BazelResponse::Artifacts() noexcept -> expected, std::string> { if (auto error_msg = Populate()) { return unexpected{*std::move(error_msg)}; } return gsl::not_null( &artifacts_); // explicit type needed for expected } auto BazelResponse::HasUpwardsSymlinks() noexcept -> expected { if (auto error_msg = Populate()) { return unexpected{*std::move(error_msg)}; } return has_upwards_symlinks_; } auto BazelResponse::Populate() noexcept -> std::optional { // Initialized only once lazily if (populated_) { return std::nullopt; } ArtifactInfos artifacts{}; auto const& action_result = output_.action_result; artifacts.reserve( static_cast(action_result.output_files_size()) + static_cast(action_result.output_file_symlinks_size()) + static_cast( action_result.output_directory_symlinks_size()) + static_cast(action_result.output_directories_size())); auto const hash_type = network_->GetHashFunction().GetType(); // collect files and store them for (auto const& file : action_result.output_files()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, file.digest()); if (not digest) { return fmt::format( "BazelResponse: failed to create artifact digest for {}", file.path()); } try { artifacts.emplace( file.path(), Artifact::ObjectInfo{.digest = *std::move(digest), .type = file.is_executable() ? ObjectType::Executable : ObjectType::File}); } catch (std::exception const& ex) { return fmt::format( "BazelResponse: unexpected failure gathering digest for " "{}:\n{}", file.path(), ex.what()); } } // in compatible mode: collect all symlinks and upload them as separate // blobs std::unordered_set link_blobs; // collect all symlinks and store them for (auto const* v_ptr : {&action_result.output_symlinks(), // DEPRECATED as of v2.1 &action_result.output_file_symlinks(), // DEPRECATED as of v2.1 &action_result.output_directory_symlinks()}) { for (auto const& link : *v_ptr) { try { // in compatible mode: track upwards symlinks has_upwards_symlinks_ = has_upwards_symlinks_ or (not ProtocolTraits::IsNative(hash_type) and not PathIsNonUpwards(link.target())); Artifact::ObjectInfo info; info.type = ObjectType::Symlink; if (ProtocolTraits::IsNative(hash_type)) { info.digest = ArtifactDigestFactory::HashDataAs( network_->GetHashFunction(), link.target()); } else { auto link_blob = ArtifactBlob::FromMemory(network_->GetHashFunction(), ObjectType::File, link.target()); if (not link_blob) { return fmt::format( "BazelResponse: failure during blob generation of " "link {}:\n{}", link.path(), std::move(link_blob).error()); } info.digest = link_blob->GetDigest(); link_blobs.insert(std::move(link_blob).value()); } artifacts.emplace(link.path(), std::move(info)); } catch (std::exception const& ex) { return fmt::format( "BazelResponse: unexpected failure gathering digest for " "{}:\n{}", link.path(), ex.what()); } } } if (ProtocolTraits::IsNative(hash_type)) { // in native mode: just collect and store tree digests for (auto const& tree : action_result.output_directories()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, tree.tree_digest()); if (not digest) { return fmt::format( "BazelResponse: failed to create artifact digest for {}", tree.path()); } ExpectsAudit(digest->IsTree()); try { artifacts.emplace( tree.path(), Artifact::ObjectInfo{.digest = *std::move(digest), .type = ObjectType::Tree}); } catch (std::exception const& ex) { return fmt::format( "BazelResponse: unexpected failure gathering digest for " "{}:\n{}", tree.path(), ex.what()); } } artifacts_ = std::move(artifacts); populated_ = true; return std::nullopt; } // Upload received symlinks. if (not network_->UploadBlobs(std::move(link_blobs))) { return fmt::format("BazelResponse: failure uploading link blobs\n"); } // obtain tree digests for output directories std::vector tree_digests{}; tree_digests.reserve( gsl::narrow(action_result.output_directories_size())); for (auto const& directory : action_result.output_directories()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, directory.tree_digest()); if (not digest) { return std::move(digest).error(); } tree_digests.push_back(*std::move(digest)); } // collect root digests from trees and store them auto reader = network_->CreateReader(); int pos = 0; for (auto const& tree_blob : reader.ReadOrdered(tree_digests)) { try { std::optional tree; if (auto const content = tree_blob.ReadContent()) { tree = BazelMsgFactory::MessageFromString( *content); } if (not tree) { return fmt::format( "BazelResponse: failed to create Tree for {}", tree_blob.GetDigest().hash()); } // The server does not store the Directory messages it just // has sent us as part of the Tree message. If we want to be // able to use the Directories as inputs for actions, we // have to upload them manually. auto upload_result = UploadTreeMessageDirectories(*tree); if (not upload_result) { auto error = fmt::format("BazelResponse: {}", std::move(upload_result).error()); Logger::Log(LogLevel::Trace, error); return error; } auto [tree_digest, tree_has_upwards_symlinks] = std::move(upload_result).value(); has_upwards_symlinks_ = has_upwards_symlinks_ or tree_has_upwards_symlinks; artifacts.emplace( action_result.output_directories(pos).path(), Artifact::ObjectInfo{.digest = std::move(tree_digest), .type = ObjectType::Tree}); } catch (std::exception const& ex) { return fmt::format( "BazelResponse: unexpected failure gathering digest for " "{}:\n{}", tree_blob.GetDigest().hash(), ex.what()); } ++pos; } artifacts_ = std::move(artifacts); populated_ = true; return std::nullopt; } auto BazelResponse::UploadTreeMessageDirectories(bazel_re::Tree const& tree) const -> expected, std::string> { auto const upload_callback = [&network = *network_](std::unordered_set&& blobs) -> bool { return network.UploadBlobs(std::move(blobs)); }; auto const hash_function = network_->GetHashFunction(); auto rootdir_result = ProcessDirectoryMessage(hash_function, tree.root()); if (not rootdir_result) { return unexpected{std::move(rootdir_result).error()}; } auto [rootdir_blob, rootdir_link_blobs, has_upwards_symlinks] = std::move(rootdir_result).value(); // Upload symlinks from root Directory message. auto rootdir_digest = rootdir_blob.GetDigest(); std::unordered_set blobs{std::move(rootdir_blob)}; for (auto link_blob : rootdir_link_blobs) { auto link_digest = link_blob.GetDigest(); if (not UpdateContainerAndUpload(&blobs, std::move(link_blob), /*exception_is_fatal=*/false, upload_callback)) { return unexpected{fmt::format( "failed to upload blobs for the Tree with the root digest {}", rootdir_digest.hash())}; } } for (auto const& subdir : tree.children()) { // store or upload blob, taking maximum transfer size into account auto subdir_result = ProcessDirectoryMessage(hash_function, subdir); if (not subdir_result) { return unexpected{std::move(subdir_result).error()}; } auto [subdir_blob, subdir_link_blobs, subdir_upwards_symlinks] = std::move(subdir_result).value(); has_upwards_symlinks = has_upwards_symlinks or subdir_upwards_symlinks; // Upload subdir Directory message. auto subdir_digest = subdir_blob.GetDigest(); if (not UpdateContainerAndUpload(&blobs, std::move(subdir_blob), /*exception_is_fatal=*/false, upload_callback)) { return unexpected{fmt::format( "failed to upload blobs for the Tree with the root digest {}", rootdir_digest.hash())}; } // Upload symlinks from root Directory message as explicit blobs. for (auto link_blob : subdir_link_blobs) { auto link_digest = link_blob.GetDigest(); if (not UpdateContainerAndUpload(&blobs, std::move(link_blob), /*exception_is_fatal=*/false, upload_callback)) { return unexpected{ fmt::format("failed to upload blobs for the Tree with the " "root digest {}", subdir_digest.hash())}; } } } // upload any remaining blob if (not std::invoke(upload_callback, std::move(blobs))) { return unexpected{fmt::format( "failed to upload blobs for the Tree with the root digest {}", rootdir_digest.hash())}; } return std::make_pair(rootdir_digest, has_upwards_symlinks); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bazel_response.hpp000066400000000000000000000117721516554100600333630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP #include #include #include #include #include // std::move, std:pair #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" #include "src/utils/cpp/expected.hpp" /// \brief Bazel implementation of the abstract Execution Response. /// Access Bazel execution output data and obtain a Bazel Artifact. class BazelResponse final : public IExecutionResponse { friend class BazelAction; public: auto Status() const noexcept -> StatusCode final { return output_.status.ok() ? StatusCode::Success : StatusCode::Failed; } auto HasStdErr() const noexcept -> bool final { return IsDigestNotEmpty(output_.action_result.stderr_digest()); } auto HasStdOut() const noexcept -> bool final { return IsDigestNotEmpty(output_.action_result.stdout_digest()); } auto StdErr() noexcept -> std::string final { return ReadStringBlob(output_.action_result.stderr_digest()); } auto StdOut() noexcept -> std::string final { return ReadStringBlob(output_.action_result.stdout_digest()); } auto StdErrDigest() noexcept -> std::optional final { auto digest = ArtifactDigestFactory::FromBazel( network_->GetHashFunction().GetType(), output_.action_result.stderr_digest()); if (digest) { return *digest; } return std::nullopt; } auto StdOutDigest() noexcept -> std::optional final { auto digest = ArtifactDigestFactory::FromBazel( network_->GetHashFunction().GetType(), output_.action_result.stdout_digest()); if (digest) { return *digest; } return std::nullopt; } auto ExitCode() const noexcept -> int final { return output_.action_result.exit_code(); } auto IsCached() const noexcept -> bool final { return output_.cached_result; }; auto ExecutionDuration() noexcept -> double final { return output_.duration; }; auto ActionDigest() const noexcept -> std::string const& final { return action_id_; } auto Artifacts() noexcept -> expected, std::string> final; auto HasUpwardsSymlinks() noexcept -> expected final; private: std::string action_id_; std::shared_ptr const network_; BazelExecutionClient::ExecutionOutput output_{}; ArtifactInfos artifacts_; bool has_upwards_symlinks_ = false; // only tracked in compatible mode bool populated_ = false; explicit BazelResponse(std::string action_id, std::shared_ptr network, BazelExecutionClient::ExecutionOutput output) : action_id_{std::move(action_id)}, network_{std::move(network)}, output_{std::move(output)} {} [[nodiscard]] auto ReadStringBlob(bazel_re::Digest const& id) noexcept -> std::string; [[nodiscard]] static auto IsDigestNotEmpty(bazel_re::Digest const& id) -> bool { return id.size_bytes() != 0; } /// \brief Populates the stored data, once. /// \returns Error message on failure, nullopt on success. [[nodiscard]] auto Populate() noexcept -> std::optional; /// \brief Tries to upload the tree rot and subdirectories. Performs also a /// symlinks check. /// \returns Pair of ArtifactDigest of root tree and flag signaling the /// presence of any upwards symlinks on success, error message on failure. [[nodiscard]] auto UploadTreeMessageDirectories( bazel_re::Tree const& tree) const -> expected, std::string>; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp000066400000000000000000000240441516554100600340610ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP #include #include #include #include #include #include #include #include // std::move #include #include "google/bytestream/bytestream.grpc.pb.h" #include "google/bytestream/bytestream.pb.h" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/incremental_reader.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// Implements client side for google.bytestream.ByteStream service. class ByteStreamClient { public: class IncrementalReader { friend class ByteStreamClient; public: /// \brief Read next chunk of data. /// \returns empty string if stream finished and std::nullopt on error. [[nodiscard]] auto Next() -> std::optional { google::bytestream::ReadResponse response{}; if (reader_->Read(&response)) { return std::move(*response.mutable_data()); } if (not finished_) { auto status = reader_->Finish(); if (not status.ok()) { logger_->Emit(LogLevel::Debug, "{}: {}", static_cast(status.error_code()), status.error_message()); return std::nullopt; } finished_ = true; } return std::string{}; } private: Logger const* logger_; grpc::ClientContext ctx_; std::unique_ptr> reader_; bool finished_{false}; IncrementalReader( gsl::not_null const& stub, std::string const& instance_name, ArtifactDigest const& digest, Logger const* logger) : logger_{logger} { google::bytestream::ReadRequest request{}; request.set_resource_name( ByteStreamUtils::ReadRequest::ToString(instance_name, digest)); reader_ = stub->Read(&ctx_, request); } }; explicit ByteStreamClient(std::string const& server, Port port, gsl::not_null const& auth) noexcept { stub_ = google::bytestream::ByteStream::NewStub( CreateChannelWithCredentials(server, port, auth)); } [[nodiscard]] auto IncrementalRead( std::string const& instance_name, ArtifactDigest const& digest) const noexcept -> IncrementalReader { return IncrementalReader{stub_.get(), instance_name, digest, &logger_}; } [[nodiscard]] auto Read(std::string const& instance_name, ArtifactDigest const& digest, TmpDir::Ptr const& temp_space) const noexcept -> std::optional { auto temp_file = TmpDir::CreateFile(temp_space, digest.hash()); if (temp_file == nullptr) { return std::nullopt; } auto reader = IncrementalRead(instance_name, digest); try { std::ofstream stream{temp_file->GetPath(), std::ios_base::binary}; auto data = reader.Next(); while (data.has_value() and not data->empty() and stream.good()) { stream << *data; data = reader.Next(); } if (not stream.good() or not data.has_value()) { return std::nullopt; } } catch (...) { return std::nullopt; } auto blob = ArtifactBlob::FromTempFile( HashFunction{digest.GetHashType()}, digest.IsTree() ? ObjectType::Tree : ObjectType::File, std::move(temp_file)); if (not blob.has_value() or blob->GetDigest() != digest) { return std::nullopt; } return *std::move(blob); } [[nodiscard]] auto Write(std::string const& instance_name, ArtifactBlob const& blob) const noexcept -> bool { thread_local static std::string uuid{}; if (uuid.empty()) { auto id = CreateProcessUniqueId(); if (not id) { logger_.Emit(LogLevel::Debug, "Failed creating process unique id."); return false; } uuid = CreateUUIDVersion4(*id); } try { grpc::ClientContext ctx; google::bytestream::WriteResponse response{}; auto writer = stub_->Write(&ctx, &response); auto const resource_name = ByteStreamUtils::WriteRequest::ToString( instance_name, uuid, blob.GetDigest()); google::bytestream::WriteRequest request{}; request.set_resource_name(resource_name); request.mutable_data()->reserve(ByteStreamUtils::kChunkSize); auto const to_read = blob.ReadIncrementally(ByteStreamUtils::kChunkSize); if (not to_read.has_value()) { logger_.Emit( LogLevel::Error, "ByteStreamClient: Failed to create a reader for {}:\n{}", request.resource_name(), to_read.error()); return false; } std::size_t pos = 0; for (auto it = to_read->begin(); it != to_read->end();) { auto const chunk = *it; if (not chunk.has_value()) { logger_.Emit( LogLevel::Error, "ByteStreamClient: Failed to read data for {}:\n{}", request.resource_name(), chunk.error()); return false; } *request.mutable_data() = *chunk; request.set_write_offset(static_cast(pos)); request.set_finish_write(pos + chunk->size() >= blob.GetContentSize()); if (writer->Write(request)) { pos += chunk->size(); ++it; } else { // According to the docs, quote: // If there is an error or the connection is broken during // the `Write()`, the client should check the status of the // `Write()` by calling `QueryWriteStatus()` and continue // writing from the returned `committed_size`. auto const committed_size = QueryWriteStatus(request.resource_name()); if (committed_size <= 0) { logger_.Emit( LogLevel::Warning, "broken stream for upload to resource name {}", request.resource_name()); return false; } pos = gsl::narrow(committed_size); it = to_read->make_iterator(pos); } } if (not writer->WritesDone()) { logger_.Emit(LogLevel::Warning, "broken stream for upload to resource name {}", request.resource_name()); return false; } auto status = writer->Finish(); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return false; } if (gsl::narrow(response.committed_size()) != blob.GetContentSize()) { logger_.Emit( LogLevel::Warning, "Committed size {} is different from the original one {}.", response.committed_size(), blob.GetContentSize()); return false; } return true; } catch (...) { logger_.Emit(LogLevel::Warning, "Caught exception in Write"); return false; } } private: std::unique_ptr stub_; Logger logger_{"ByteStreamClient"}; [[nodiscard]] auto QueryWriteStatus( std::string const& resource_name) const noexcept -> std::int64_t { grpc::ClientContext ctx; google::bytestream::QueryWriteStatusRequest request{}; request.set_resource_name(resource_name); google::bytestream::QueryWriteStatusResponse response{}; stub_->QueryWriteStatus(&ctx, request, &response); return response.committed_size(); } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/config.cpp000066400000000000000000000103331516554100600305030ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/config.hpp" #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" auto RemoteExecutionConfig::Builder::Build() const noexcept -> expected { // To not duplicate default arguments in builder, create a default config // and copy arguments from there. RemoteExecutionConfig const default_config{}; // Set remote endpoint. auto remote_address = default_config.remote_address; if (remote_address_raw_.has_value()) { remote_address = ParseAddress(*remote_address_raw_); if (not remote_address) { return unexpected{ fmt::format("Failed to set remote endpoint address {}", nlohmann::json(*remote_address_raw_).dump())}; } } // Set cache endpoint. auto cache_address = default_config.cache_address; if (cache_address_raw_.has_value()) { cache_address = ParseAddress(*cache_address_raw_); // Cache endpoint can be in the usual "host:port" or the literal // "local", so we only fail if we cannot parse a non-"local" string, // because parsing a "local" literal will correctly return a nullopt. bool const is_parsed = cache_address.has_value(); bool const is_local = *cache_address_raw_ == "local"; if (not is_local and not is_parsed) { return unexpected{ fmt::format("Failed to set cache endpoint address {}", nlohmann::json(*cache_address_raw_).dump())}; } } else { // If cache address not explicitly set, it defaults to remote address. cache_address = remote_address; } // Set dispatch info. auto dispatch = default_config.dispatch; if (dispatch_file_.has_value()) { try { if (auto dispatch_info = FileSystemManager::ReadFile(*dispatch_file_)) { auto parsed = ParseDispatch(*dispatch_info); if (not parsed) { return unexpected{parsed.error()}; } dispatch = *std::move(parsed); } else { return unexpected{ fmt::format("Failed to read json file {}", nlohmann::json(*dispatch_file_).dump())}; } } catch (std::exception const& e) { return unexpected{fmt::format( "Assigning the endpoint configuration failed with:\n{}", e.what())}; } } // Set platform properties. auto platform_properties = default_config.platform_properties; for (auto const& property : platform_properties_raw_) { if (auto pair = ParseProperty(property)) { try { platform_properties.insert_or_assign(pair->first, pair->second); } catch (std::exception const& e) { return unexpected{fmt::format("Failed to insert property {}", nlohmann::json(property).dump())}; } } else { return unexpected{fmt::format("Adding platform property {} failed.", nlohmann::json(property).dump())}; } } return RemoteExecutionConfig{ .remote_address = std::move(remote_address), .remote_instance_name = remote_instance_name_, .dispatch = std::move(dispatch), .cache_address = std::move(cache_address), .platform_properties = std::move(platform_properties)}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/config.hpp000066400000000000000000000071161516554100600305150ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP #include #include #include #include #include #include "src/buildtool/common/remote/remote_common.hpp" #include "src/utils/cpp/expected.hpp" struct RemoteExecutionConfig final { class Builder; // Server address of remote execution. std::optional const remote_address; // Instance name of the remote execution std::string const remote_instance_name; // Server dispatch data std::vector const dispatch; // Server address of cache endpoint for rebuild. std::optional const cache_address; // Platform properties for execution. ExecutionProperties const platform_properties; }; class RemoteExecutionConfig::Builder final { public: // Set remote execution and cache address. If it fails to parse during // config build, will reset the fields. auto SetRemoteAddress(std::optional address) noexcept -> Builder& { remote_address_raw_ = std::move(address); return *this; } // Set remote execution instance name. auto SetRemoteInstanceName(std::string name) noexcept -> Builder& { remote_instance_name_ = std::move(name); return *this; } // Set remote-execution dispatch property list filename. auto SetRemoteExecutionDispatch( std::optional filename) noexcept -> Builder& { dispatch_file_ = std::move(filename); return *this; } // Set specific cache address. If it fails to parse during config build, // will reset the field. auto SetCacheAddress(std::optional address) noexcept -> Builder& { cache_address_raw_ = std::move(address); return *this; } // Set platform properties given as "key:val" strings. auto SetPlatformProperties(std::vector properties) noexcept -> Builder& { platform_properties_raw_ = std::move(properties); return *this; } /// \brief Parse the set data to finalize creation of RemoteExecutionConfig. /// \return RemoteExecutionConfig on success, an error string on failure. [[nodiscard]] auto Build() const noexcept -> expected; private: // Server address of remote execution; needs parsing. std::optional remote_address_raw_; // Instance name of the remote execution std::string remote_instance_name_; // Server dispatch data file; needs parsing. std::optional dispatch_file_; // Server address of cache endpoint for rebuild; needs parsing. std::optional cache_address_raw_; // Platform properties for execution; needs parsing. std::vector platform_properties_raw_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/remote/context.hpp000066400000000000000000000024711516554100600307330ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONTEXT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONTEXT_HPP #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/execution_api/remote/config.hpp" /// \brief Aggregate of remote execution-related entities. /// \note No field is be stored as const ref to avoid binding to temporaries. struct RemoteContext final { gsl::not_null const auth; gsl::not_null const retry_config; gsl::not_null const exec_config; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONTEXT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/000077500000000000000000000000001516554100600263635ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/TARGETS000066400000000000000000000031121516554100600274140ustar00rootroot00000000000000{ "mr_git_api": { "type": ["@", "rules", "CC", "library"] , "name": ["mr_git_api"] , "hdrs": ["mr_git_api.hpp"] , "srcs": ["mr_git_api.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": [ ["src/buildtool/execution_api/git", "git_api"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] } , "mr_local_api": { "type": ["@", "rules", "CC", "library"] , "name": ["mr_local_api"] , "hdrs": ["mr_local_api.hpp"] , "srcs": ["mr_local_api.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": [ ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "expected"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/mr_git_api.cpp000066400000000000000000000043641516554100600312100ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include #include #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" MRGitApi::MRGitApi( gsl::not_null const& repo_config, gsl::not_null const& native_storage_config, StorageConfig const* compatible_storage_config, IExecutionApi const* compatible_local_api) noexcept : repo_config_{repo_config}, native_storage_config_{native_storage_config}, compat_storage_config_{compatible_storage_config}, compat_local_api_{compatible_local_api} {} auto MRGitApi::RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool { // in native mode: dispatch to regular GitApi if (compat_storage_config_ == nullptr) { GitApi const git_api{repo_config_}; return git_api.RetrieveToCas(artifacts_info, api); } auto compat_artifacts = RehashUtils::RehashGitDigest(artifacts_info, *native_storage_config_, *compat_storage_config_, *repo_config_); if (not compat_artifacts) { Logger::Log(LogLevel::Error, "MRGitApi: {}", std::move(compat_artifacts).error()); return false; } return compat_local_api_->RetrieveToCas(*compat_artifacts, api); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/mr_git_api.hpp000066400000000000000000000046011516554100600312070ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/storage/config.hpp" /// \brief Handles interaction between the Git CAS and another api, irrespective /// of the remote-execution protocol used. class MRGitApi final { public: MRGitApi(gsl::not_null const& repo_config, gsl::not_null const& native_storage_config, StorageConfig const* compatible_storage_config = nullptr, IExecutionApi const* compatible_local_api = nullptr) noexcept; /// \brief Passes artifacts from Git CAS to specified (remote) api. In /// compatible mode, it must rehash the native digests to be able to upload /// to a compatible remote. Expects native digests. /// \note Caller is responsible for passing vectors with artifacts of the /// same digest type. [[nodiscard]] auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool; private: gsl::not_null repo_config_; // retain references to needed storages and configs gsl::not_null native_storage_config_; StorageConfig const* compat_storage_config_; // an api accessing compatible storage, used purely to communicate with a // compatible remote; only instantiated if in compatible mode IExecutionApi const* compat_local_api_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/mr_local_api.cpp000066400000000000000000000141741516554100600315170ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include #include #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" MRLocalApi::MRLocalApi( gsl::not_null const& native_context, gsl::not_null const& native_local_api, LocalContext const* compatible_context, IExecutionApi const* compatible_local_api) noexcept : native_context_{native_context}, compat_context_{compatible_context}, native_local_api_{native_local_api}, compat_local_api_{compatible_local_api} {} auto MRLocalApi::RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* /*alternative*/) const noexcept -> bool { // This method can legitimately be called with both native and // compatible digests when in compatible mode, therefore we need to // interrogate the hash type of the input. // we need at least one digest to interrogate the hash type if (artifacts_info.empty()) { return true; // nothing to do } // native artifacts get dispatched to native local api if (ProtocolTraits::IsNative(artifacts_info[0].digest.GetHashType())) { return native_local_api_->RetrieveToPaths(artifacts_info, output_paths); } // compatible digests get dispatched to compatible local api if (compat_local_api_ == nullptr) { Logger::Log(LogLevel::Error, "MRLocalApi: Unexpected digest type provided"); return false; } return compat_local_api_->RetrieveToPaths(artifacts_info, output_paths); } auto MRLocalApi::RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool { // return immediately if being passed the same api if (this == &api) { return true; } // in native mode: dispatch directly to native local api if (compat_local_api_ == nullptr) { return native_local_api_->RetrieveToCas(artifacts_info, api); } // in compatible mode: if compatible hashes passed, dispatch them to // compatible local api if (not artifacts_info.empty() and not ProtocolTraits::IsNative(artifacts_info[0].digest.GetHashType())) { return compat_local_api_->RetrieveToCas(artifacts_info, api); } auto compat_artifacts = RehashUtils::RehashDigest( artifacts_info, *native_context_->storage_config, *compat_context_->storage_config, /*apis=*/std::nullopt); // all parts of git trees are present locally if (not compat_artifacts) { Logger::Log(LogLevel::Error, "MRLocalApi: {}", std::move(compat_artifacts).error()); return false; } return compat_local_api_->RetrieveToCas(*compat_artifacts, api); } auto MRLocalApi::Upload(std::unordered_set&& blobs, bool skip_find_missing) const noexcept -> bool { // in native mode, dispatch to native local api if (compat_local_api_ == nullptr) { return native_local_api_->Upload(std::move(blobs), skip_find_missing); } // in compatible mode, dispatch to compatible local api return compat_local_api_->Upload(std::move(blobs), skip_find_missing); } auto MRLocalApi::IsAvailable(ArtifactDigest const& digest) const noexcept -> bool { // This method can legitimately be called with both native and // compatible digests when in compatible mode, therefore we need to // interrogate the hash type of the input. // a native digest gets dispatched to native local api if (ProtocolTraits::IsNative(digest.GetHashType())) { return native_local_api_->IsAvailable(digest); } // compatible digests get dispatched to compatible local api if (compat_local_api_ == nullptr) { Logger::Log(LogLevel::Warning, "MRLocalApi: unexpected digest type provided"); return false; } return compat_local_api_->IsAvailable(digest); } auto MRLocalApi::GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set { // This method can legitimately be called with both native and // compatible digests when in compatible mode, therefore we need to // interrogate the hash type of the input. // we need at least one digest to interrogate the hash type if (digests.empty()) { return {}; // nothing to do } // native digests get dispatched to native local api if (ProtocolTraits::IsNative(digests.begin()->GetHashType())) { return native_local_api_->GetMissingDigests(digests); } // compatible digests get dispatched to compatible local api if (compat_local_api_ == nullptr) { Logger::Log(LogLevel::Warning, "MRLocalApi: Unexpected digest type provided"); return {}; } return compat_local_api_->GetMissingDigests(digests); } auto MRLocalApi::GetHashType() const noexcept -> HashFunction::Type { return compat_local_api_ == nullptr ? native_local_api_->GetHashType() : compat_local_api_->GetHashType(); } auto MRLocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr { return native_local_api_->GetTempSpace(); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/serve/mr_local_api.hpp000066400000000000000000000152411516554100600315200ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_LOCAL_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_LOCAL_API_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief Multi-repo-specific implementation of the abstract Execution API. /// Handles interaction between a native storage and a remote, irrespective of /// the remote protocol used. In compatible mode, both native and compatible /// storages are available. class MRLocalApi final : public IExecutionApi { public: /// \brief Construct a new MRLocalApi object. In native mode only the native /// storage in instantiated (hence behaving like regular LocalApi), while in /// compatible mode both storages are instantiated. MRLocalApi(gsl::not_null const& native_context, gsl::not_null const& native_local_api, LocalContext const* compatible_context = nullptr, IExecutionApi const* compatible_local_api = nullptr) noexcept; /// \brief Not supported. [[nodiscard]] auto CreateAction( ArtifactDigest const& /*root_digest*/, std::vector const& /*command*/, std::string const& /*cwd*/, std::vector const& /*output_files*/, std::vector const& /*output_dirs*/, std::map const& /*env_vars*/, std::map const& /*properties*/, bool /*force_legacy*/) const noexcept -> IExecutionAction::Ptr final { // Execution not supported return nullptr; } /// \brief Stages artifacts from CAS to the file system. /// Handles both native and compatible artifacts. Dispatches to appropriate /// local api instance based on digest hash type. Alternative api is never /// used. [[nodiscard]] auto RetrieveToPaths( std::vector const& artifacts_info, std::vector const& output_paths, IExecutionApi const* /*alternative*/) const noexcept -> bool final; [[nodiscard]] auto RetrieveToFds( std::vector const& /*artifacts_info*/, std::vector const& /*fds*/, bool /*raw_tree*/, IExecutionApi const* /*alternative*/) const noexcept -> bool final { // Retrieval to file descriptors not supported return false; } /// \brief Passes artifacts from native CAS to specified api. Handles both /// native and compatible digests. In compatible mode, if passed native /// digests it must rehash them to be able to upload to a compatible remote. /// \note Caller is responsible for passing vectors with artifacts of the /// same digest type. For simplicity, this method takes the first digest of /// the vector as representative for figuring out hash function type. [[nodiscard]] auto RetrieveToCas( std::vector const& artifacts_info, IExecutionApi const& api) const noexcept -> bool final; [[nodiscard]] auto RetrieveToMemory( Artifact::ObjectInfo const& /*artifact_info*/) const noexcept -> std::optional final { // Retrieval to memory not supported return std::nullopt; } /// \brief Uploads artifacts from local CAS into specified api. Dispatches /// the blobs to the appropriate local api instance based on used protocol. /// \note Caller is responsible for passing vectors with artifacts of the /// same digest type. [[nodiscard]] auto Upload(std::unordered_set&& blobs, bool skip_find_missing) const noexcept -> bool final; [[nodiscard]] auto UploadTree( std::vector const& /*artifacts*/) const noexcept -> std::optional final { // Upload tree not supported -- only used in execution return std::nullopt; } /// \brief Check availability of an artifact in CAS. Handles both native and /// compatible digests. Dispatches to appropriate local api instance based /// on digest hash type. [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool final; /// \brief Check availability of artifacts in CAS. Handles both native and /// compatible digests. Dispatches to appropriate local api instance based /// on hash type of digests. /// \note The caller is responsible for passing vectors with digests of the /// same type. For simplicity, this method takes the first digest of the /// vector as representative for figuring out hash function type. [[nodiscard]] auto GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set final; [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; private: // retain local context references to have direct access to storages gsl::not_null native_context_; LocalContext const* compat_context_; // local api accessing native storage; all artifacts must pass through it gsl::not_null native_local_api_; // local api accessing compatible storage, used purely to communicate with // a compatible remote; only instantiated if in compatible mode IExecutionApi const* compat_local_api_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_LOCAL_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/000077500000000000000000000000001516554100600263775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/TARGETS000066400000000000000000000037711516554100600274430ustar00rootroot00000000000000{ "subobject": { "type": ["@", "rules", "CC", "library"] , "name": ["subobject"] , "hdrs": ["subobject.hpp"] , "srcs": ["subobject.cpp"] , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "api_bundle"] ] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "execution_api", "utils"] } , "outputscheck": { "type": ["@", "rules", "CC", "library"] , "name": ["outputscheck"] , "hdrs": ["outputscheck.hpp"] , "deps": [ ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "bazel_types"] ] , "stage": ["src", "buildtool", "execution_api", "utils"] } , "rehash_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["utils"] , "hdrs": ["rehash_utils.hpp"] , "srcs": ["rehash_utils.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "execution_api", "utils"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/outputscheck.hpp000066400000000000000000000071601516554100600316350ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_OUTPUTSCHECK_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_OUTPUTSCHECK_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "src/buildtool/common/bazel_types.hpp" [[nodiscard]] static inline auto ActionResultContainsExpectedOutputs( const bazel_re::ActionResult& result, const std::vector& expected_files, const std::vector& expected_dirs) noexcept -> bool { try { std::set actual_output_files{}; for (auto const& file : result.output_files()) { actual_output_files.emplace(file.path()); } for (auto const& file : result.output_file_symlinks()) { actual_output_files.emplace(file.path()); } for (auto const& file : result.output_directory_symlinks()) { actual_output_files.emplace(file.path()); } if (not std::all_of(expected_files.begin(), expected_files.end(), [&actual_output_files](auto const& expected) { return actual_output_files.contains(expected); })) { return false; } std::set actual_output_dirs{}; for (auto const& dir : result.output_directories()) { actual_output_dirs.emplace(dir.path()); } return std::all_of(expected_dirs.begin(), expected_dirs.end(), [&actual_output_dirs](auto const& expected) { return actual_output_dirs.contains(expected); }); } catch (...) { return false; } } // overload for treating file and directory paths alike [[nodiscard]] static inline auto ActionResultContainsExpectedOutputs( const bazel_re::ActionResult& result, const std::vector& expected_paths) noexcept -> bool { try { std::set actual_output_paths{}; for (auto const& file : result.output_files()) { actual_output_paths.emplace(file.path()); } for (auto const& file : result.output_file_symlinks()) { actual_output_paths.emplace(file.path()); } for (auto const& file : result.output_directory_symlinks()) { actual_output_paths.emplace(file.path()); } for (auto const& dir : result.output_directories()) { actual_output_paths.emplace(dir.path()); } return std::all_of(expected_paths.begin(), expected_paths.end(), [&actual_output_paths](auto const& expected) { return actual_output_paths.contains(expected); }); } catch (...) { return false; } } #endif #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_OUTPUTSCHECK_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/rehash_utils.cpp000066400000000000000000000327601516554100600316050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include #include // std::size_t #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" namespace RehashUtils { auto ReadRehashedDigest(ArtifactDigest const& digest, StorageConfig const& source_config, StorageConfig const& target_config, bool from_git) noexcept -> expected, std::string> { // check for mapping file in all generations std::size_t generation = 0; std::optional rehash_id_file = std::nullopt; auto const compat_hash_type = target_config.hash_function.GetType(); for (; generation < source_config.num_generations; ++generation) { auto path = StorageUtils::GetRehashIDFile(source_config, compat_hash_type, digest.hash(), from_git, generation); if (FileSystemManager::Exists(path)) { rehash_id_file = std::move(path); break; // found the generation } } if (rehash_id_file) { // read id file auto compat_obj_str = FileSystemManager::ReadFile(*rehash_id_file); if (not compat_obj_str) { return unexpected{fmt::format("failed to read rehash id file {}", rehash_id_file->string())}; } // get artifact object from content auto compat_obj = Artifact::ObjectInfo::FromString(compat_hash_type, *compat_obj_str); if (not compat_obj) { // handle nullopt value explicitly return unexpected{ fmt::format("failed to read rehashed artifact from id file {}", rehash_id_file->string())}; } // ensure the id file is in generation 0 for future calls if (generation != 0) { auto dest_id_file = StorageUtils::GetRehashIDFile( source_config, compat_hash_type, digest.hash(), from_git); auto ok = FileSystemManager::CreateFileHardlink(*rehash_id_file, dest_id_file); if (not ok) { auto const& err = ok.error(); if (err != std::errc::too_many_links) { return unexpected{ fmt::format("failed to link rehash id file {}:\n{} {}", dest_id_file.string(), err.value(), err.message())}; } // if too many links reported, write id file ourselves if (not StorageUtils::WriteTreeIDFile(dest_id_file, *compat_obj_str)) { return unexpected{ fmt::format("failed to write rehash id file {}", dest_id_file.string())}; } } } return compat_obj; // not dereferenced to assist type deduction in // variant } // no mapping file found return std::optional{std::nullopt}; } auto StoreRehashedDigest(ArtifactDigest const& source_digest, ArtifactDigest const& target_digest, ObjectType obj_type, StorageConfig const& source_config, StorageConfig const& target_config, bool from_git) noexcept -> std::optional { // write mapping auto const rehash_id_file = StorageUtils::GetRehashIDFile(source_config, target_config.hash_function.GetType(), source_digest.hash(), from_git); if (not StorageUtils::WriteTreeIDFile( rehash_id_file, Artifact::ObjectInfo{.digest = target_digest, .type = obj_type} .ToString())) { return fmt::format("failed to write rehash id to file {}", rehash_id_file.string()); } return std::nullopt; // a-ok } namespace { template TReadCallback> [[nodiscard]] auto RehashDigestImpl( std::vector const& infos, StorageConfig const& source_config, StorageConfig const& target_config, TReadCallback read_callback, bool from_git) -> expected, std::string> { auto const target_storage = Storage::Create(&target_config); BazelMsgFactory::FileStoreFunc store_file = [&target_storage](std::filesystem::path const& data, bool is_exec) -> std::optional { return target_storage.CAS().StoreBlob(data, is_exec); }; BazelMsgFactory::BlobStoreFunc store_from_git = [&target_storage]( std::variant const& data, bool is_exec) -> std::optional { return std::visit( [&target_storage, is_exec](auto const& d) { return target_storage.CAS().StoreBlob(d, is_exec); }, data); }; BazelMsgFactory::TreeStoreFunc store_dir = [&cas = target_storage.CAS()](std::string const& content) { return cas.StoreTree(content); }; BazelMsgFactory::SymlinkStoreFunc store_symlink = [&cas = target_storage.CAS()](std::string const& content) { return cas.StoreBlob(content); }; BazelMsgFactory::RehashedDigestReadFunc read_rehashed = [&source_config, &target_config, from_git](ArtifactDigest const& digest) -> expected, std::string> { return RehashUtils::ReadRehashedDigest( digest, source_config, target_config, from_git); }; BazelMsgFactory::RehashedDigestStoreFunc store_rehashed = [&source_config, &target_config, from_git]( ArtifactDigest const& source_digest, ArtifactDigest const& target_digest, ObjectType obj_type) -> std::optional { return RehashUtils::StoreRehashedDigest(source_digest, target_digest, obj_type, source_config, target_config, from_git); }; // collect the native blobs and rehash them as compatible to be able to // check what is missing in the other api std::vector compat_artifacts; compat_artifacts.reserve(infos.size()); for (auto const& source_object : infos) { // check if we know already the compatible digest auto cached_obj = read_rehashed(source_object.digest); if (not cached_obj) { return unexpected(std::move(cached_obj).error()); } if (*cached_obj) { // add object to the vector of compatible artifacts compat_artifacts.emplace_back(std::move(cached_obj)->value()); continue; } // process object; trees need to be handled appropriately if (IsTreeObject(source_object.type)) { // get the directory digest auto target_tree = ProtocolTraits::IsNative(target_config.hash_function.GetType()) ? BazelMsgFactory::CreateGitTreeDigestFromDirectory( source_object.digest, read_callback, store_file, store_dir, store_symlink, read_rehashed, store_rehashed) : BazelMsgFactory::CreateDirectoryDigestFromGitTree( source_object.digest, read_callback, store_from_git, store_dir, store_symlink, read_rehashed, store_rehashed); if (not target_tree) { return unexpected(std::move(target_tree).error()); } // add object to the vector of compatible artifacts compat_artifacts.emplace_back(Artifact::ObjectInfo{ .digest = *std::move(target_tree), .type = ObjectType::Tree}); } else { // blobs can be directly rehashed auto data = read_callback(source_object.digest, source_object.type); if (not data) { return unexpected(fmt::format("failed to get path of entry {}", source_object.digest.hash())); } auto const is_exec = IsExecutableObject(source_object.type); auto target_blob = from_git ? store_from_git(*data, is_exec) : store_file(*data, is_exec); if (not target_blob) { return unexpected(fmt::format("failed to rehash entry {}", source_object.digest.hash())); } // cache the digest association if (auto error_msg = store_rehashed( source_object.digest, *target_blob, source_object.type)) { return unexpected(*std::move(error_msg)); } // add object to the vector of compatible artifacts compat_artifacts.emplace_back(Artifact::ObjectInfo{ .digest = *std::move(target_blob), .type = source_object.type}); } } return compat_artifacts; } } // namespace auto RehashDigest(std::vector const& digests, StorageConfig const& source_config, StorageConfig const& target_config, std::optional> apis) -> expected, std::string> { auto const source_storage = Storage::Create(&source_config); auto read = [&source_storage, &apis]( ArtifactDigest const& digest, ObjectType type) -> std::optional { if (apis) { auto const& local = *(*apis)->local; auto const& remote = *(*apis)->remote; if (not local.IsAvailable(digest) and not remote.RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, .type = type}}, local)) { return std::nullopt; } } return IsTreeObject(type) ? source_storage.CAS().TreePath(digest) : source_storage.CAS().BlobPath( digest, IsExecutableObject(type)); }; return RehashDigestImpl(digests, source_config, target_config, std::move(read), /*from_git=*/false); } auto RehashGitDigest(std::vector const& digests, StorageConfig const& source_config, StorageConfig const& target_config, RepositoryConfig const& repo_config) -> expected, std::string> { auto read = [&repo_config](ArtifactDigest const& digest, ObjectType type) -> std::optional { return repo_config.ReadBlobFromGitCAS( digest.hash(), /*is_symlink=*/IsSymlinkObject(type)); }; return RehashDigestImpl(digests, source_config, target_config, std::move(read), /*from_git=*/true); } auto Rehasher::Rehash(Artifact::ObjectInfo const& info) const -> expected { auto rehashed = RehashUtils::RehashDigest( std::vector{info}, source_, target_, apis_); if (not rehashed) { return unexpected(rehashed.error()); } return rehashed.value()[0]; } } // namespace RehashUtils just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/rehash_utils.hpp000066400000000000000000000104271516554100600316060ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" namespace RehashUtils { /// \brief Get a corresponding known object from a different local CAS, as /// stored in a mapping file, if exists. /// \param digest Source digest. /// \param source_config Storage config corresponding to source digest. /// \param target_config Storage config corresponding to target digest. /// \param from_git Specify if source digest comes from a Git location instead /// of CAS. /// \returns The target artifact info on successfully reading an existing /// mapping file, nullopt if no mapping file exists, or the error message on /// failure. [[nodiscard]] auto ReadRehashedDigest(ArtifactDigest const& digest, StorageConfig const& source_config, StorageConfig const& target_config, bool from_git = false) noexcept -> expected, std::string>; /// \brief Write the mapping file linking two digests hashing the same content. /// \param source_digest Source digest. /// \param target_digest Target digest. /// \param obj_type Object type of the content represented by the two digests. /// \param source_config Storage config corresponding to source digest. /// \param target_config Storage config corresponding to target digest. /// \param from_git Specify if source digest comes from a Git location instead /// of CAS. /// \returns nullopt on success, error message on failure. [[nodiscard]] auto StoreRehashedDigest(ArtifactDigest const& source_digest, ArtifactDigest const& target_digest, ObjectType obj_type, StorageConfig const& source_config, StorageConfig const& target_config, bool from_git = false) noexcept -> std::optional; [[nodiscard]] auto RehashDigest( std::vector const& digests, StorageConfig const& source_config, StorageConfig const& target_config, std::optional> apis) -> expected, std::string>; [[nodiscard]] auto RehashGitDigest( std::vector const& digests, StorageConfig const& source_config, StorageConfig const& target_config, RepositoryConfig const& repo_config) -> expected, std::string>; class Rehasher { public: Rehasher(StorageConfig source_config, StorageConfig target_config, std::optional> apis) : source_{std::move(source_config)}, target_{std::move(target_config)}, apis_{std::move(apis)} {} [[nodiscard]] auto Rehash(Artifact::ObjectInfo const& info) const -> expected; private: StorageConfig const source_; StorageConfig const target_; std::optional> const apis_; }; } // namespace RehashUtils #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/subobject.cpp000066400000000000000000000123741516554100600310720ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/utils/subobject.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" auto RetrieveSubPathId(Artifact::ObjectInfo object_info, ApiBundle const& apis, const std::filesystem::path& sub_path) -> std::optional { std::filesystem::path sofar{}; for (auto const& segment : sub_path) { if (object_info.type != ObjectType::Tree) { Logger::Log(LogLevel::Warning, "Non-tree found at path '{}', cannot follow to '{}'", sofar.string(), segment.string()); break; } auto data = apis.remote->RetrieveToMemory(object_info); if (not data) { Logger::Log(LogLevel::Error, "Failed to retrieve artifact {} at path '{}'", object_info.ToString(), sofar.string()); return std::nullopt; } if (not ProtocolTraits::IsNative(apis.remote->GetHashType())) { auto directory = BazelMsgFactory::MessageFromString(*data); if (not directory) { Logger::Log(LogLevel::Warning, "Failed to parse directory message at path '{}'", sofar.string()); break; } std::optional new_object_info{}; if (not TreeReaderUtils::ReadObjectInfos( *directory, [&new_object_info, &segment]( std::filesystem::path const& path, Artifact::ObjectInfo&& info) { if (path == segment) { new_object_info = std::move(info); } return true; })) { Logger::Log(LogLevel::Warning, "Failed to process directory message at path '{}'", sofar.string()); break; } if (not new_object_info) { Logger::Log(LogLevel::Warning, "Entry {} not found at path '{}'", segment.string(), sofar.string()); break; } object_info = *new_object_info; } else { auto entries = GitRepo::ReadTreeData( *data, object_info.digest.hash(), [](auto const& /*unused*/) { return true; }, /*is_hex_id=*/true); if (not entries) { Logger::Log(LogLevel::Warning, "Failed to parse tree {} at path '{}'", object_info.ToString(), sofar.string()); break; } std::optional new_object_info{}; if (not TreeReaderUtils::ReadObjectInfos( *entries, [&new_object_info, &segment]( std::filesystem::path const& path, Artifact::ObjectInfo&& info) { if (path == segment) { new_object_info = std::move(info); } return true; })) { Logger::Log(LogLevel::Warning, "Failed to process tree entries at path '{}'", sofar.string()); break; } if (not new_object_info) { Logger::Log(LogLevel::Warning, "Entry {} not found at path '{}'", segment.string(), sofar.string()); break; } object_info = *new_object_info; } sofar /= segment; } return object_info; } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_api/utils/subobject.hpp000066400000000000000000000023331516554100600310710ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_SUBOBJECT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_SUBOBJECT_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" auto RetrieveSubPathId(Artifact::ObjectInfo object_info, ApiBundle const& apis, const std::filesystem::path& sub_path) -> std::optional; #endif // BOOTSTRAP_BUILD_TOOL #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_SUBOBJECT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/000077500000000000000000000000001516554100600257335ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/dag/000077500000000000000000000000001516554100600264665ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/dag/TARGETS000066400000000000000000000010571516554100600275250ustar00rootroot00000000000000{ "dag": { "type": ["@", "rules", "CC", "library"] , "name": ["dag"] , "hdrs": ["dag.hpp"] , "srcs": ["dag.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "type_safe_arithmetic"] ] , "stage": ["src", "buildtool", "execution_engine", "dag"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/dag/dag.cpp000066400000000000000000000214211516554100600277250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_engine/dag/dag.hpp" #include #include #include auto DependencyGraph::CreateOutputArtifactNodes( std::string const& action_id, std::vector const& file_paths, std::vector const& dir_paths, bool is_tree_action, bool is_tree_overlay_action) -> std::pair, std::vector> { if (is_tree_overlay_action) { // create tree-overlay artifact auto artifact = ArtifactDescription::CreateTreeOverlay(action_id).ToArtifact(); auto const node_id = AddArtifact(std::move(artifact)); return std::make_pair(std::vector{}, std::vector{ {{}, &(*artifact_nodes_[node_id])}}); } if (is_tree_action) { // create tree artifact auto artifact = ArtifactDescription::CreateTree(action_id).ToArtifact(); auto const node_id = AddArtifact(std::move(artifact)); return std::make_pair(std::vector{}, std::vector{ {{}, &(*artifact_nodes_[node_id])}}); } // create action artifacts auto node_creator = [this, &action_id](auto* nodes, auto const& paths) { for (auto const& artifact_path : paths) { auto artifact = ArtifactDescription::CreateAction( action_id, std::filesystem::path{artifact_path}) .ToArtifact(); auto const node_id = AddArtifact(std::move(artifact)); nodes->emplace_back(NamedArtifactNodePtr{ artifact_path, &(*artifact_nodes_[node_id])}); } }; std::vector file_nodes{}; file_nodes.reserve(file_paths.size()); node_creator(&file_nodes, file_paths); std::vector dir_nodes{}; dir_nodes.reserve(dir_paths.size()); node_creator(&dir_nodes, dir_paths); return std::make_pair(std::move(file_nodes), std::move(dir_nodes)); } auto DependencyGraph::CreateInputArtifactNodes( ActionDescription::inputs_t const& inputs) -> std::optional> { std::vector nodes{}; for (auto const& [local_path, artifact_desc] : inputs) { auto artifact = artifact_desc.ToArtifact(); auto const node_id = AddArtifact(std::move(artifact)); nodes.push_back({local_path, &(*artifact_nodes_[node_id])}); } return nodes; } auto DependencyGraph::CreateActionNode(Action const& action) noexcept -> DependencyGraph::ActionNode* { if (action.IsTreeAction() or action.IsTreeOverlayAction() or not action.Command().empty()) { auto const node_id = AddAction(action); return &(*action_nodes_[node_id]); } return nullptr; } auto DependencyGraph::LinkNodePointers( std::vector const& output_files, std::vector const& output_dirs, gsl::not_null const& action_node, std::vector const& input_nodes) noexcept -> bool { for (auto const& named_file : output_files) { if (not named_file.node->AddBuilderActionNode(action_node) or not action_node->AddOutputFile(named_file)) { return false; } } for (auto const& named_dir : output_dirs) { if (not named_dir.node->AddBuilderActionNode(action_node) or not action_node->AddOutputDir(named_dir)) { return false; } } for (auto const& named_node : input_nodes) { if (not named_node.node->AddConsumerActionNode(action_node) or not action_node->AddDependency(named_node)) { return false; } } action_node->NotifyDoneLinking(); return true; } auto DependencyGraph::Add(std::vector const& actions) -> bool { return std::all_of( actions.begin(), actions.end(), [this](auto const& action) { return AddAction(action); }); } auto DependencyGraph::Add(std::vector const& actions) -> bool { return std::all_of( actions.begin(), actions.end(), [this](auto const& action) { return AddAction(*action); }); } auto DependencyGraph::AddArtifact(ArtifactDescription const& description) -> ArtifactIdentifier { auto artifact = description.ToArtifact(); auto id = artifact.Id(); [[maybe_unused]] auto const node_id = AddArtifact(std::move(artifact)); return id; } auto DependencyGraph::AddAction(ActionDescription const& description) -> bool { auto output_nodes = CreateOutputArtifactNodes( description.Id(), description.OutputFiles(), description.OutputDirs(), description.GraphAction().IsTreeAction(), description.GraphAction().IsTreeOverlayAction()); auto* action_node = CreateActionNode(description.GraphAction()); auto input_nodes = CreateInputArtifactNodes(description.Inputs()); if (action_node == nullptr or not input_nodes.has_value() or (output_nodes.first.empty() and output_nodes.second.empty())) { return false; } return LinkNodePointers( output_nodes.first, output_nodes.second, action_node, *input_nodes); } auto DependencyGraph::AddAction(Action const& a) noexcept -> DependencyGraph::ActionNodeIdentifier { auto const& id = a.Id(); auto const action_it = action_ids_.find(id); if (action_it != action_ids_.end()) { return action_it->second; } action_nodes_.emplace_back(ActionNode::Create(a)); ActionNodeIdentifier node_id{action_nodes_.size() - 1}; action_ids_[id] = node_id; return node_id; } auto DependencyGraph::AddAction(Action&& a) noexcept -> DependencyGraph::ActionNodeIdentifier { auto const& id = a.Id(); auto const action_it = action_ids_.find(id); if (action_it != action_ids_.end()) { return action_it->second; } action_nodes_.emplace_back(ActionNode::Create(std::move(a))); ActionNodeIdentifier node_id{action_nodes_.size() - 1}; action_ids_[id] = node_id; return node_id; } auto DependencyGraph::AddArtifact(Artifact const& a) noexcept -> DependencyGraph::ArtifactNodeIdentifier { auto const& id = a.Id(); auto const artifact_it = artifact_ids_.find(id); if (artifact_it != artifact_ids_.end()) { return artifact_it->second; } artifact_nodes_.emplace_back(ArtifactNode::Create(a)); ArtifactNodeIdentifier node_id{artifact_nodes_.size() - 1}; artifact_ids_[id] = node_id; return node_id; } auto DependencyGraph::AddArtifact(Artifact&& a) noexcept -> DependencyGraph::ArtifactNodeIdentifier { auto id = a.Id(); auto const artifact_it = artifact_ids_.find(id); if (artifact_it != artifact_ids_.end()) { return artifact_it->second; } artifact_nodes_.emplace_back(ArtifactNode::Create(std::move(a))); ArtifactNodeIdentifier node_id{artifact_nodes_.size() - 1}; artifact_ids_[id] = node_id; return node_id; } auto DependencyGraph::ArtifactIdentifiers() const noexcept -> std::unordered_set { std::unordered_set ids; std::transform( std::begin(artifact_ids_), std::end(artifact_ids_), std::inserter(ids, std::begin(ids)), [](auto const& artifact_id_pair) { return artifact_id_pair.first; }); return ids; } auto DependencyGraph::ArtifactNodeWithId(ArtifactIdentifier const& id) const noexcept -> DependencyGraph::ArtifactNode const* { auto it_to_artifact = artifact_ids_.find(id); if (it_to_artifact == artifact_ids_.end()) { return nullptr; } return &(*artifact_nodes_[it_to_artifact->second]); } auto DependencyGraph::ActionNodeWithId(ActionIdentifier const& id) const noexcept -> DependencyGraph::ActionNode const* { auto it_to_action = action_ids_.find(id); if (it_to_action == action_ids_.end()) { return nullptr; } return &(*action_nodes_[it_to_action->second]); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/dag/dag.hpp000066400000000000000000000476341516554100600277500ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/type_safe_arithmetic.hpp" /// \brief Plain DirectedAcyclicGraph. /// Additional properties (e.g. bipartiteness) can be encoded in nodes. /// Deliberately not using \ref DirectedAcyclicGraph::Node anywhere to avoid /// vtable lookups. For now, it does not hold any data. class DirectedAcyclicGraph { public: /// \brief Abstract class for DAG nodes. /// \tparam TContent Type of content. /// \tparam TOther Type of neighboring nodes. /// TODO: once we have hashes, require sub classes to implement generating /// IDs depending on its unique content. template class Node { public: using OtherNode = TOther; using OtherNodePtr = gsl::not_null; explicit Node(TContent&& content) noexcept : content_{std::move(content)} {} // NOLINTNEXTLINE(modernize-pass-by-value) explicit Node(TContent const& content) noexcept : content_{content} {} Node(TContent const& content, std::vector const& parents, std::vector const& children) noexcept : content_{content}, parents_{parents}, children_{children} {} // No copies and no moves are allowed. The only way to "copy" an // instance is to copy its raw pointer. Node(Node const&) = delete; Node(Node&&) = delete; auto operator=(Node const&) -> Node& = delete; auto operator=(Node&&) -> Node& = delete; ~Node() = default; [[nodiscard]] auto Content() const& noexcept -> TContent const& { return content_; } [[nodiscard]] auto Content() && noexcept -> TContent { return std::move(content_); } [[nodiscard]] auto Parents() const& noexcept -> std::vector const& { return parents_; } [[nodiscard]] auto Children() const& noexcept -> std::vector const& { return children_; } auto AddParent(OtherNodePtr const& parent) noexcept -> bool { parents_.push_back(parent); return true; } auto AddParent(OtherNodePtr&& parent) noexcept -> bool { parents_.push_back(std::move(parent)); return true; } auto AddChild(OtherNodePtr const& child) noexcept -> bool { children_.push_back(child); return true; } auto AddChild(OtherNodePtr&& child) noexcept -> bool { children_.push_back(std::move(child)); return true; } private: TContent content_{}; std::vector parents_{}; std::vector children_{}; }; /// \brief Lock-free class for basic traversal state data /// Provides the following atomic operations: /// - Retrieve (previous) state and mark as discovered, which will allow us /// to know whether we should queue a visit the node or not at the same /// time that we mark that its visit should not be queued by other threads, /// since it is being queued by the current caller to this method or it has /// already been queued by a previous caller. /// Note that "discovered" refers to "queued for visit" here. /// - Retrieve (previous) state and mark as queued to be processed, which /// will allow us to ensure that processing a node is queued at most once. class NodeTraversalState { public: NodeTraversalState() noexcept = default; NodeTraversalState(NodeTraversalState const&) = delete; NodeTraversalState(NodeTraversalState&&) = delete; auto operator=(NodeTraversalState const&) -> NodeTraversalState& = delete; auto operator=(NodeTraversalState&&) -> NodeTraversalState& = delete; ~NodeTraversalState() noexcept = default; /// \brief Sets traversal state as discovered /// \returns True if it was already discovered, false otherwise /// Note: this is an atomic, lock-free operation [[nodiscard]] auto GetAndMarkDiscovered() noexcept -> bool { return std::atomic_exchange(&has_been_discovered_, true); } /// \brief Sets traversal state as queued to be processed /// \returns True if it was already queued to be processed, false /// otherwise /// Note: this is an atomic, lock-free operation [[nodiscard]] auto GetAndMarkQueuedToBeProcessed() noexcept -> bool { return std::atomic_exchange(&is_queued_to_be_processed_, true); } /// \brief Check if a node is required to be processed or not [[nodiscard]] auto IsRequired() const noexcept -> bool { return is_required_; } /// \brief Mark node as required to be executed /// Note: this should be upon node discovery (visit) while traversing /// the graph void MarkRequired() noexcept { is_required_ = true; } private: std::atomic has_been_discovered_{false}; std::atomic is_queued_to_be_processed_{false}; std::atomic is_required_{false}; }; }; class DependencyGraph : DirectedAcyclicGraph { public: // Forward declaration class ArtifactNode; // Node identifier for actions struct ActionNodeIdentifierTag : TypeSafeArithmeticTag {}; using ActionNodeIdentifier = TypeSafeArithmetic; // Node identifier for artifacts struct ArtifactNodeIdentifierTag : TypeSafeArithmeticTag {}; using ArtifactNodeIdentifier = TypeSafeArithmetic; /// \brief Class for traversal state data specific for ActionNode's /// Provides the following atomic operations (listed on the public methods): class ActionNodeTraversalState : public NodeTraversalState { public: ActionNodeTraversalState() noexcept = default; ActionNodeTraversalState(ActionNodeTraversalState const&) = delete; ActionNodeTraversalState(ActionNodeTraversalState&&) = delete; auto operator=(ActionNodeTraversalState const&) -> ActionNodeTraversalState& = delete; auto operator=(ActionNodeTraversalState&&) -> ActionNodeTraversalState& = delete; ~ActionNodeTraversalState() noexcept = default; /// \brief Acknowledge that a dependency was made available and return /// whether the action is ready to be executed [[nodiscard]] auto NotifyAvailableDepAndCheckReady() noexcept -> bool { return std::atomic_fetch_sub(&unavailable_deps_, 1) == 1; } /// \brief Check whether the action can be now executed or not. /// Note: checking state without modifying (unlike /// NotifyAvailableDepAndCheckReady()) is useful in the case that when /// the action node is visited all its dependencies were already /// available [[nodiscard]] auto IsReady() const noexcept -> bool { return unavailable_deps_ == 0; } /// \brief Initialise number of unavailable dependencies /// \param[in] count Number of unavailable dependencies /// Note: this method should be called previous to the start of the /// traversal (once the action node is built) void InitUnavailableDeps(std::size_t count) noexcept { unavailable_deps_ = static_cast(count); } private: std::atomic unavailable_deps_{-1}; }; /// \brief Class for traversal state data specific for ArtifactNode's /// Provides the following atomic operations: /// - Mark the artifact in this node as available /// - Check whether the artifact in this node is available or not class ArtifactNodeTraversalState : public NodeTraversalState { public: ArtifactNodeTraversalState() noexcept = default; ArtifactNodeTraversalState(ArtifactNodeTraversalState const&) = delete; ArtifactNodeTraversalState(ArtifactNodeTraversalState&&) = delete; auto operator=(ArtifactNodeTraversalState const&) -> ArtifactNodeTraversalState& = delete; auto operator=(ArtifactNodeTraversalState&&) -> ArtifactNodeTraversalState& = delete; ~ArtifactNodeTraversalState() noexcept = default; [[nodiscard]] auto IsAvailable() const noexcept -> bool { return is_available_; } void MakeAvailable() noexcept { is_available_ = true; } private: std::atomic is_available_{false}; }; /// \brief Action node (bipartite) /// Cannot be entry. class ActionNode final : public Node { using base = Node; public: using base::base; using Ptr = std::unique_ptr; struct NamedOtherNodePtr { Action::LocalPath path; base::OtherNodePtr node; }; [[nodiscard]] static auto Create(Action const& content) noexcept -> Ptr { return std::make_unique(content); } [[nodiscard]] static auto Create(Action&& content) noexcept -> Ptr { return std::make_unique(std::move(content)); } [[nodiscard]] auto AddOutputFile( NamedOtherNodePtr const& output) noexcept -> bool { base::AddParent(output.node); output_files_.push_back(output); return true; } [[nodiscard]] auto AddOutputFile(NamedOtherNodePtr&& output) noexcept -> bool { base::AddParent(output.node); output_files_.push_back(std::move(output)); return true; } [[nodiscard]] auto AddOutputDir( NamedOtherNodePtr const& output) noexcept -> bool { base::AddParent(output.node); output_dirs_.push_back(output); return true; } [[nodiscard]] auto AddOutputDir(NamedOtherNodePtr&& output) noexcept -> bool { base::AddParent(output.node); output_dirs_.push_back(std::move(output)); return true; } [[nodiscard]] auto AddDependency( NamedOtherNodePtr const& dependency) noexcept -> bool { base::AddChild(dependency.node); dependencies_.push_back(dependency); return true; } [[nodiscard]] auto AddDependency( NamedOtherNodePtr&& dependency) noexcept -> bool { base::AddChild(dependency.node); dependencies_.push_back(std::move(dependency)); return true; } [[nodiscard]] auto OutputFiles() const& noexcept -> std::vector const& { return output_files_; } [[nodiscard]] auto OutputDirs() const& noexcept -> std::vector const& { return output_dirs_; } [[nodiscard]] auto Dependencies() const& noexcept -> std::vector const& { return dependencies_; } [[nodiscard]] auto Command() const noexcept -> std::vector const& { return Content().Command(); } [[nodiscard]] auto Env() const noexcept -> std::map const& { return Content().Env(); } [[nodiscard]] auto MayFail() const noexcept -> std::optional const& { return Content().MayFail(); } [[nodiscard]] auto TimeoutScale() const noexcept -> double { return Content().TimeoutScale(); } [[nodiscard]] auto ExecutionProperties() const noexcept -> std::map const& { return Content().ExecutionProperties(); } [[nodiscard]] auto NoCache() const noexcept -> bool { return Content().NoCache(); } [[nodiscard]] auto OutputFilePaths() const& noexcept -> std::vector { return NodePaths(output_files_); } [[nodiscard]] auto OutputDirPaths() const& noexcept -> std::vector { return NodePaths(output_dirs_); } [[nodiscard]] auto DependencyPaths() const& noexcept -> std::vector { return NodePaths(dependencies_); } // To initialise the action traversal specific data before traversing // the graph void NotifyDoneLinking() const noexcept { traversal_state_->InitUnavailableDeps(Children().size()); } [[nodiscard]] auto TraversalState() const noexcept -> ActionNodeTraversalState* { return traversal_state_.get(); } private: std::vector output_files_; std::vector output_dirs_; std::vector dependencies_; std::unique_ptr traversal_state_{ std::make_unique()}; [[nodiscard]] static auto NodePaths( std::vector const& nodes) -> std::vector { std::vector paths{nodes.size()}; std::transform( nodes.cbegin(), nodes.cend(), paths.begin(), [](auto const& named_node) { return named_node.path; }); return paths; } }; /// \brief Artifact node (bipartite) /// Can be entry or leaf and can only have single child (see \ref AddChild) class ArtifactNode final : public Node { using base = Node; public: using base::base; using typename base::OtherNode; using typename base::OtherNodePtr; using Ptr = std::unique_ptr; [[nodiscard]] static auto Create(Artifact const& content) noexcept -> Ptr { return std::make_unique(content); } [[nodiscard]] static auto Create(Artifact&& content) noexcept -> Ptr { return std::make_unique(std::move(content)); } [[nodiscard]] auto AddBuilderActionNode( OtherNodePtr const& action) noexcept -> bool { if (base::Children().empty()) { return base::AddChild(action); } Logger::Log(LogLevel::Error, "cannot set a second builder for artifact {}", ToHexString(Content().Id())); return false; } [[nodiscard]] auto AddConsumerActionNode( OtherNodePtr const& action) noexcept -> bool { return base::AddParent(action); } [[nodiscard]] auto HasBuilderAction() const noexcept -> bool { return not base::Children().empty(); } [[nodiscard]] auto BuilderActionNode() const noexcept -> ActionNode const* { return HasBuilderAction() ? base::Children()[0].get() : nullptr; } [[nodiscard]] auto TraversalState() const noexcept -> ArtifactNodeTraversalState* { return traversal_state_.get(); } private: std::unique_ptr traversal_state_{ std::make_unique()}; }; using NamedArtifactNodePtr = ActionNode::NamedOtherNodePtr; DependencyGraph() noexcept = default; // DependencyGraph should not be copyable or movable. This could be changed // in the case we want to make the graph construction to be functional DependencyGraph(DependencyGraph const&) = delete; DependencyGraph(DependencyGraph&&) = delete; auto operator=(DependencyGraph const&) -> DependencyGraph& = delete; auto operator=(DependencyGraph&&) -> DependencyGraph& = delete; ~DependencyGraph() noexcept = default; [[nodiscard]] auto Add(std::vector const& actions) -> bool; [[nodiscard]] auto Add(std::vector const& actions) -> bool; [[nodiscard]] auto AddAction(ActionDescription const& description) -> bool; [[nodiscard]] auto AddArtifact(ArtifactDescription const& description) -> ArtifactIdentifier; [[nodiscard]] auto ArtifactIdentifiers() const noexcept -> std::unordered_set; [[nodiscard]] auto ArtifactNodeWithId( ArtifactIdentifier const& id) const noexcept -> ArtifactNode const*; [[nodiscard]] auto ActionNodeWithId( ActionIdentifier const& id) const noexcept -> ActionNode const*; private: // List of action nodes we already created std::vector action_nodes_; // List of artifact nodes we already created std::vector artifact_nodes_; // Associates global action identifier to local node id std::unordered_map action_ids_; // Associates global artifact identifier to local node id std::unordered_map artifact_ids_; [[nodiscard]] auto CreateOutputArtifactNodes( std::string const& action_id, std::vector const& file_paths, std::vector const& dir_paths, bool is_tree_action, bool is_tree_overlay_action) -> std::pair, std::vector>; [[nodiscard]] auto CreateInputArtifactNodes( ActionDescription::inputs_t const& inputs) -> std::optional>; [[nodiscard]] auto CreateActionNode(Action const& action) noexcept -> ActionNode*; [[nodiscard]] static auto LinkNodePointers( std::vector const& output_files, std::vector const& output_dirs, gsl::not_null const& action_node, std::vector const& input_nodes) noexcept -> bool; [[nodiscard]] auto AddAction(Action const& a) noexcept -> ActionNodeIdentifier; [[nodiscard]] auto AddAction(Action&& a) noexcept -> ActionNodeIdentifier; [[nodiscard]] auto AddArtifact(Artifact const& a) noexcept -> ArtifactNodeIdentifier; [[nodiscard]] auto AddArtifact(Artifact&& a) noexcept -> ArtifactNodeIdentifier; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/executor/000077500000000000000000000000001516554100600275715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/executor/TARGETS000066400000000000000000000047301516554100600306310ustar00rootroot00000000000000{ "executor": { "type": ["@", "rules", "CC", "library"] , "name": ["executor"] , "hdrs": ["executor.hpp"] , "deps": [ "context" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "git_hashes_converter"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/remote", "bazel_api"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/execution_engine/dag", "dag"] , [ "src/buildtool/execution_engine/tree_operations" , "tree_operations_utils" ] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_tree"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/profile", "profile"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path_rebase"] , ["src/utils/cpp", "prefix"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_engine", "executor"] } , "context": { "type": ["@", "rules", "CC", "library"] , "name": ["context"] , "hdrs": ["context.hpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/profile", "profile"] , ["src/buildtool/progress_reporting", "progress"] ] , "stage": ["src", "buildtool", "execution_engine", "executor"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/executor/context.hpp000066400000000000000000000032251516554100600317700ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP #include #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/profile/profile.hpp" #include "src/buildtool/progress_reporting/progress.hpp" /// \brief Aggregate to be passed to graph traverser. /// \note No field is stored as const ref to avoid binding to temporaries. struct ExecutionContext final { gsl::not_null const repo_config; gsl::not_null const& apis; gsl::not_null const remote_context; gsl::not_null const statistics; gsl::not_null const progress; std::optional> const profile; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/executor/executor.hpp000066400000000000000000001532031516554100600321440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/git_hashes_converter.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/execution_engine/tree_operations/tree_operations_utils.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/profile/profile.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/utils/cpp/back_map.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path_rebase.hpp" #include "src/utils/cpp/prefix.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief Implementations for executing actions and uploading artifacts. class ExecutorImpl { public: /// \brief Computes the result of a tree-overlay action. /// \returns std::nullopt on success, nullptr on error. [[nodiscard]] static auto ExecuteTreeOverlayAction( Logger const& logger, gsl::not_null const& action, IExecutionApi const& api, gsl::not_null const& progress) -> std::optional { progress->TaskTracker().Start(action->Content().Id()); std::vector inputs = action->Dependencies(); std::sort(inputs.begin(), inputs.end(), [](auto a, auto b) { return a.path < b.path; }); logger.Emit(LogLevel::Debug, [&]() { std::ostringstream oss{}; oss << "Requested tree-overlay action is " << action->Content().Id() << "\n"; oss << "Trees to overlay:"; for (auto const& input : inputs) { oss << "\n - " << input.node->Content().Info()->digest.hash() << ":" << input.node->Content().Info()->digest.size() << ":" << ToChar(input.node->Content().Info()->type); } return oss.str(); }); auto empty_tree = TreeOperationsUtils::WriteTree( api, TreeOperationsUtils::TreeEntries{}); if (not empty_tree) { logger.Emit(LogLevel::Error, "Failed to build the empty tree:\n{}", empty_tree.error()); return nullptr; } auto tree = empty_tree.value(); for (auto const& overlay : inputs) { auto computed_overlay = TreeOperationsUtils::ComputeTreeOverlay( api, tree, *overlay.node->Content().Info(), action->Content().IsOverlayDisjoint()); if (not computed_overlay) { logger.Emit(LogLevel::Error, "Tree-overlay computation failed:\n{}", computed_overlay.error()); return nullptr; } tree = computed_overlay.value(); } auto const& tree_artifact = action->OutputDirs()[0].node->Content(); bool failed_inputs = false; for (auto const& [local_path, artifact] : inputs) { failed_inputs |= artifact->Content().Info()->failed; } tree_artifact.SetObjectInfo( tree.digest, ObjectType::Tree, failed_inputs); progress->TaskTracker().Stop(action->Content().Id()); return std::nullopt; } /// \brief Execute action and obtain response. /// \returns std::nullopt for actions without response (e.g., tree actions). /// \returns nullptr on error. [[nodiscard]] static auto ExecuteAction( Logger const& logger, gsl::not_null const& action, IExecutionApi const& api, ExecutionProperties const& merged_properties, gsl::not_null const& remote_context, std::chrono::milliseconds const& timeout, IExecutionAction::CacheFlag cache_flag, gsl::not_null const& stats, gsl::not_null const& progress) noexcept -> std::optional { try { if (action->Content().IsTreeOverlayAction()) { return ExecuteTreeOverlayAction(logger, action, api, progress); } auto const& inputs = action->Dependencies(); auto const tree_action = action->Content().IsTreeAction(); logger.Emit(LogLevel::Trace, [&inputs, tree_action]() { std::ostringstream oss{}; oss << "execute " << (tree_action ? "tree " : "") << "action" << std::endl; for (auto const& [local_path, artifact] : inputs) { auto const& info = artifact->Content().Info(); oss << fmt::format( " - needs {} {}", local_path, info ? info->ToString() : std::string{"[???]"}) << std::endl; } return oss.str(); }); auto const root_digest = CreateRootDigest(api, inputs); if (not root_digest) { logger.Emit( LogLevel::Error, "failed to create root digest for input artifacts."); return nullptr; } if (tree_action) { auto const& tree_artifact = action->OutputDirs()[0].node->Content(); bool failed_inputs = false; for (auto const& [local_path, artifact] : inputs) { failed_inputs |= artifact->Content().Info()->failed; } tree_artifact.SetObjectInfo( *root_digest, ObjectType::Tree, failed_inputs); return std::nullopt; } // do not count statistics for rebuilder fetching from cache if (cache_flag != IExecutionAction::CacheFlag::FromCacheOnly) { progress->TaskTracker().Start(action->Content().Id()); stats->IncrementActionsQueuedCounter(); } // get the alternative endpoint auto alternative_api = GetAlternativeEndpoint(merged_properties, remote_context, api.GetHashType(), api.GetTempSpace()); if (alternative_api) { if (not api.ParallelRetrieveToCas( std::vector{ Artifact::ObjectInfo{*root_digest, ObjectType::Tree, /* failed= */ false}}, *alternative_api, /* jobs= */ 1, /* use_blob_splitting= */ true)) { logger.Emit(LogLevel::Error, "Failed to sync tree {} to dispatch endpoint", root_digest->hash()); return nullptr; } } auto base = action->Content().Cwd(); auto cwd_relative_output_files = RebasePathStringsRelativeTo(base, action->OutputFilePaths()); auto cwd_relative_output_dirs = RebasePathStringsRelativeTo(base, action->OutputDirPaths()); auto remote_action = (alternative_api ? *alternative_api : api) .CreateAction(*root_digest, action->Command(), base, cwd_relative_output_files, cwd_relative_output_dirs, action->Env(), merged_properties); if (remote_action == nullptr) { logger.Emit(LogLevel::Error, "failed to create action for execution."); return nullptr; } // set action options remote_action->SetCacheFlag(cache_flag); remote_action->SetTimeout(timeout); // execute action auto result = remote_action->Execute(&logger); // process result if (result) { // in compatible mode, check that all artifacts are valid if (not ProtocolTraits::IsNative(api.GetHashType())) { auto upwards_symlinks_check = result->HasUpwardsSymlinks(); if (not upwards_symlinks_check) { logger.Emit(LogLevel::Error, upwards_symlinks_check.error()); return nullptr; } if (upwards_symlinks_check.value()) { logger.Emit( LogLevel::Error, "Executed action produced invalid outputs -- " "upwards symlinks"); return nullptr; } } // if alternative endpoint used, transfer any missing blobs if (alternative_api) { auto const artifacts = result->Artifacts(); if (not artifacts) { logger.Emit(LogLevel::Error, artifacts.error()); return nullptr; } std::vector object_infos{}; object_infos.reserve(artifacts.value()->size()); for (auto const& [path, info] : *artifacts.value()) { object_infos.emplace_back(info); } if (not alternative_api->RetrieveToCas(object_infos, api)) { logger.Emit(LogLevel::Warning, "Failed to retrieve back artifacts from " "dispatch endpoint"); } } } return result; } catch (std::exception const& ex) { logger.Emit(LogLevel::Error, "Unexpectedly failed to execute action with:\n{}", ex.what()); return nullptr; } } /// \brief Ensures the artifact is available to the CAS, either checking /// that its existing digest corresponds to that of an object already /// available or by uploading it if there is no digest in the artifact. In /// the later case, the new digest is saved in the artifact /// \param[in] artifact The artifact to process. /// \returns True if artifact is available at the point of return, false /// otherwise [[nodiscard]] static auto VerifyOrUploadArtifact( Logger const& logger, gsl::not_null const& artifact, gsl::not_null const& repo_config, ApiBundle const& apis) noexcept -> bool { auto const object_info_opt = artifact->Content().Info(); auto const file_path_opt = artifact->Content().FilePath(); // If there is no object info and no file path, the artifact can not be // processed: it means its definition is ill-formed or that it is the // output of an action, in which case it shouldn't have reached here if (not object_info_opt and not file_path_opt) { logger.Emit(LogLevel::Error, "artifact {} can not be processed.", ToHexString(artifact->Content().Id())); return false; } // If the artifact has digest, we check that an object with this digest // is available to the execution API if (object_info_opt) { logger.Emit(LogLevel::Trace, [&object_info_opt]() { std::ostringstream oss{}; oss << fmt::format("upload KNOWN artifact: {}", object_info_opt->ToString()) << std::endl; return oss.str(); }); if (not apis.remote->IsAvailable(object_info_opt->digest)) { // Check if requested artifact is available in local CAS and // upload to remote CAS in case it is. if (apis.local->IsAvailable(object_info_opt->digest) and apis.local->RetrieveToCas({*object_info_opt}, *apis.remote)) { return true; } if (not VerifyOrUploadKnownArtifact( logger, *apis.remote, artifact->Content().Repository(), repo_config, *object_info_opt)) { logger.Emit( LogLevel::Error, "artifact {} should be present in CAS but is missing.", ToHexString(artifact->Content().Id())); return false; } } return true; } // Otherwise, we upload the new file to make it available to the // execution API // Note that we can be sure now that file_path_opt has a value and // that the path stored is relative to the workspace dir, so we need to // prepend it logger.Emit(LogLevel::Trace, [&file_path_opt]() { std::ostringstream oss{}; oss << fmt::format("upload LOCAL artifact: {}", file_path_opt->string()) << std::endl; return oss.str(); }); auto repo = artifact->Content().Repository(); auto new_info = UploadFile(*apis.remote, repo, repo_config, *file_path_opt); if (not new_info) { logger.Emit(LogLevel::Error, "artifact in {} could not be uploaded to CAS.", file_path_opt->string()); return false; } // And we save the digest object type in the artifact artifact->Content().SetObjectInfo(*new_info, false); return true; } /// \brief Uploads the content of a git tree recursively to the CAS. It is /// first checked which elements of a directory are not available in the /// CAS and the missing elements are uploaded accordingly. This ensures the /// invariant that if a git tree is known to the CAS all its content is also /// existing in the CAS. /// \param[in] api The remote execution API of the CAS. /// \param[in] tree The git tree to be uploaded. /// \returns True if the upload was successful, False in case of any error. [[nodiscard]] static auto VerifyOrUploadTree(Logger const& logger, IExecutionApi const& api, GitTree const& tree) noexcept -> bool { // create list of digests for batch check of CAS availability using ElementType = typename GitTree::entries_t::value_type; auto const back_map = BackMap::Make( &tree, [](ElementType const& entry) { return ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, entry.second->Hash(), *entry.second->Size(), entry.second->IsTree()); }); if (back_map == nullptr) { return false; } logger.Emit(LogLevel::Trace, [&tree]() { std::ostringstream oss{}; oss << "upload directory content of " << tree.FileRootHash() << std::endl; for (auto const& [path, entry] : tree) { oss << fmt::format(" - {}: {}", path, entry->Hash()) << std::endl; } return oss.str(); }); // find missing digests auto const missing_digests = api.GetMissingDigests(back_map->GetKeys()); auto const missing_entries = back_map->IterateReferences(&missing_digests); // process missing trees for (auto const& [_, value] : missing_entries) { auto const entry = value->second; if (entry->IsTree()) { auto const& tree = entry->Tree(); if (not tree or not VerifyOrUploadTree(logger, api, *tree)) { return false; } } } // upload missing entries (blobs or trees) HashFunction const hash_function{api.GetHashType()}; std::unordered_set container; for (auto const& [digest, value] : missing_entries) { auto const entry = value->second; auto content = entry->RawData(); if (not content.has_value()) { return false; } auto blob = ArtifactBlob::FromMemory( hash_function, entry->Type(), *std::move(content)); if (not blob.has_value()) { return false; } // store and/or upload blob, taking into account the maximum // transfer size if (not UpdateContainerAndUpload( &container, *std::move(blob), /*exception_is_fatal=*/true, [&api](std::unordered_set&& blobs) { return api.Upload(std::move(blobs), /*skip_find_missing=*/true); })) { return false; } } // upload remaining blobs return api.Upload(std::move(container), /*skip_find_missing=*/true); } /// \brief Lookup blob via digest in local git repositories and upload. /// \param api The endpoint used for uploading /// \param repo The global repository name, the artifact belongs to /// \param info The info of the object /// \param hash The git-sha1 hash of the object /// \returns true on success [[nodiscard]] static auto VerifyOrUploadGitArtifact( Logger const& logger, IExecutionApi const& api, std::string const& repo, gsl::not_null const& repo_config, Artifact::ObjectInfo const& info, std::string const& hash) noexcept -> bool { std::optional content; if (info.digest.IsTree()) { // if known tree is not available, recursively upload its content auto tree = ReadGitTree(repo, repo_config, hash); if (not tree) { logger.Emit( LogLevel::Error, "failed to read git tree {}", hash); return false; } if (not VerifyOrUploadTree(logger, api, *tree)) { logger.Emit(LogLevel::Error, "failed to verifyorupload git tree {} [{}]", tree->FileRootHash(), hash); return false; } content = tree->RawData(); } else { // if known blob is not available, read and upload it content = ReadGitBlob( repo, repo_config, hash, IsSymlinkObject(info.type)); } if (not content) { logger.Emit(LogLevel::Error, "failed to get content"); return false; } auto blob = ArtifactBlob::FromMemory( HashFunction{api.GetHashType()}, info.type, *std::move(content)); if (not blob.has_value()) { logger.Emit(LogLevel::Error, "failed to create ArtifactBlob"); return false; } return api.Upload({*std::move(blob)}, /*skip_find_missing=*/true); } [[nodiscard]] static auto ReadGitBlob( std::string const& repo, gsl::not_null const& repo_config, std::string const& hash, bool is_symlink) noexcept -> std::optional { std::optional blob{}; if (auto const* ws_root = repo_config->WorkspaceRoot(repo)) { // try to obtain blob from local workspace's Git CAS, if any blob = ws_root->ReadBlob(hash, is_symlink); } if (not blob) { // try to obtain blob from global Git CAS, if any blob = repo_config->ReadBlobFromGitCAS(hash, is_symlink); } return blob; } [[nodiscard]] static auto ReadGitTree( std::string const& repo, gsl::not_null const& repo_config, std::string const& hash) noexcept -> std::optional { std::optional tree{}; if (auto const* ws_root = repo_config->WorkspaceRoot(repo)) { // try to obtain tree from local workspace's Git CAS, if any tree = ws_root->ReadTree(hash); } if (not tree) { // try to obtain tree from global Git CAS, if any tree = repo_config->ReadTreeFromGitCAS(hash); } return tree; } /// \brief Lookup blob via digest in local git repositories and upload. /// \param api The endpoint used for uploading /// \param repo The global repository name, the artifact belongs to /// \param repo_config Configuration specifying the workspace root /// \param info The info of the object /// \returns true on success [[nodiscard]] static auto VerifyOrUploadKnownArtifact( Logger const& logger, IExecutionApi const& api, std::string const& repo, gsl::not_null const& repo_config, Artifact::ObjectInfo const& info) noexcept -> bool { if (not ProtocolTraits::IsNative(api.GetHashType())) { auto opt = GitHashesConverter::Instance().GetGitEntry(info.digest.hash()); if (opt) { auto const& [git_sha1_hash, comp_repo] = *opt; return VerifyOrUploadGitArtifact( logger, api, comp_repo, repo_config, info, git_sha1_hash); } return false; } return VerifyOrUploadGitArtifact( logger, api, repo, repo_config, info, info.digest.hash()); } /// \brief Lookup file via path in local workspace root and upload. /// \param api The endpoint used for uploading /// \param repo The global repository name, the artifact belongs to /// \param repo_config Configuration specifying the workspace root /// \param file_path The path of the file to be read /// \returns The computed object info on success [[nodiscard]] static auto UploadFile( IExecutionApi const& api, std::string const& repo, gsl::not_null const& repo_config, std::filesystem::path const& file_path) noexcept -> std::optional { auto const* ws_root = repo_config->WorkspaceRoot(repo); if (ws_root == nullptr) { return std::nullopt; } auto const object_type = ws_root->BlobType(file_path); if (not object_type) { return std::nullopt; } auto content = ws_root->ReadContent(file_path); if (not content.has_value()) { return std::nullopt; } auto blob = ArtifactBlob::FromMemory( HashFunction{api.GetHashType()}, *object_type, *std::move(content)); if (not blob.has_value()) { return std::nullopt; } auto digest = blob->GetDigest(); if (not api.Upload({*std::move(blob)})) { return std::nullopt; } return Artifact::ObjectInfo{.digest = std::move(digest), .type = *object_type}; } /// \brief Add digests and object type to artifact nodes for all outputs of /// the action that was run void static SaveObjectInfo( IExecutionResponse::ArtifactInfos const& artifacts, gsl::not_null const& action, bool fail_artifacts) noexcept { auto base = action->Content().Cwd(); for (auto const& [name, node] : action->OutputFiles()) { node->Content().SetObjectInfo( artifacts.at(RebasePathStringRelativeTo(base, name)), fail_artifacts); } for (auto const& [name, node] : action->OutputDirs()) { node->Content().SetObjectInfo( artifacts.at(RebasePathStringRelativeTo(base, name)), fail_artifacts); } } /// \brief Create root tree digest for input artifacts. /// \param api The endpoint required for uploading /// \param artifacts The artifacts to create the root tree digest from [[nodiscard]] static auto CreateRootDigest( IExecutionApi const& api, std::vector const& artifacts) -> std::optional { if (artifacts.size() == 1 and (artifacts[0].path == "." or artifacts[0].path.empty())) { auto const& info = artifacts[0].node->Content().Info(); if (info and IsTreeObject(info->type)) { // Artifact list contains single tree with path "." or "". Reuse // the existing tree artifact by returning its digest. return info->digest; } } return api.UploadTree(artifacts); } /// \brief Check that all outputs expected from the action description /// are present in the artifacts map and of the expected type template [[nodiscard]] static auto CheckOutputsExist( IExecutionResponse::ArtifactInfos const& artifacts, std::vector const& outputs, std::string base) -> bool { return std::all_of( outputs.begin(), outputs.end(), [&artifacts, &base](Action::LocalPath const& output) { auto it = artifacts.find(RebasePathStringRelativeTo(base, output)); if (it != artifacts.end()) { auto const& type = it->second.type; if constexpr (kIsTree) { return IsTreeObject(type) or IsSymlinkObject(type); } else { return IsFileObject(type) or IsSymlinkObject(type); } } return false; }); } /// \brief Parse response and write object info to DAG's artifact nodes. /// \returns false on non-zero exit code or if output artifacts are missing [[nodiscard]] static auto ParseResponse( Logger const& logger, IExecutionResponse::Ptr const& response, gsl::not_null const& action, gsl::not_null const& stats, gsl::not_null const& progress, bool count_as_executed = false) -> bool { logger.Emit(LogLevel::Trace, "finished execution"); if (not response) { logger.Emit(LogLevel::Trace, "response is empty"); PrintError(logger, action, progress); return false; } if (not count_as_executed and response->IsCached()) { logger.Emit(LogLevel::Trace, " - served from cache"); stats->IncrementActionsCachedCounter(); } else { stats->IncrementActionsExecutedCounter(); } progress->TaskTracker().Stop(action->Content().Id()); PrintInfo(logger, action, response); bool action_failed = false; bool should_fail_outputs = false; for (auto const& [local_path, node] : action->Dependencies()) { should_fail_outputs |= node->Content().Info()->failed; } if (response->ExitCode() != 0) { if (action->MayFail()) { should_fail_outputs = true; action_failed = true; } else { logger.Emit(LogLevel::Error, "action returned non-zero exit code {}", response->ExitCode()); PrintError(logger, action, progress); return false; } } auto const artifacts = response->Artifacts(); if (not artifacts) { logger.Emit(LogLevel::Error, artifacts.error()); return false; } auto output_files = action->OutputFilePaths(); auto output_dirs = action->OutputDirPaths(); std::sort(output_files.begin(), output_files.end()); std::sort(output_dirs.begin(), output_dirs.end()); if (artifacts.value()->empty() or not CheckOutputsExist( *artifacts.value(), output_files, action->Content().Cwd()) or not CheckOutputsExist( *artifacts.value(), output_dirs, action->Content().Cwd())) { logger.Emit(LogLevel::Error, [&]() { std::ostringstream message{}; if (action_failed) { message << *(action->MayFail()) << " (exit code " << response->ExitCode() << ")\nMoreover "; } message << "action executed with missing outputs.\nAction " "outputs should be the following artifacts:"; for (auto const& output : output_files) { message << "\n - file: " << output; } for (auto const& output : output_dirs) { message << "\n - dir: " << output; } return message.str(); }); PrintError(logger, action, progress); return false; } SaveObjectInfo(*artifacts.value(), action, should_fail_outputs); if (action_failed) { logger.Emit(LogLevel::Warning, [&]() { std::ostringstream message{}; auto base = action->Content().Cwd(); message << *(action->MayFail()) << " (exit code " << response->ExitCode() << "); outputs:"; for (auto const& name : output_files) { message << "\n - " << nlohmann::json(name).dump() << " " << artifacts.value() ->at(RebasePathStringRelativeTo(base, name)) .ToString(); } for (auto const& name : output_dirs) { message << "\n - " << nlohmann::json(name).dump() << " " << artifacts.value() ->at(RebasePathStringRelativeTo(base, name)) .ToString(); } return message.str(); }); } return true; } /// \brief Write out if response is empty and otherwise, write out /// standard error/output if they are present void static PrintInfo( Logger const& logger, gsl::not_null const& action, IExecutionResponse::Ptr const& response) { if (not response) { logger.Emit(LogLevel::Error, "response is empty"); return; } auto const has_err = response->HasStdErr(); auto const has_out = response->HasStdOut(); auto build_message = [has_err, has_out, &action, &response]() { using namespace std::string_literals; auto message = ""s; bool has_both = has_err and has_out; if (has_err or has_out) { if (has_both) { message += "Output"s; } else { message += has_out ? "Stdout"s : "Stderr"s; } message += " of command "; } message += nlohmann::json(action->Command()).dump() + " in environment " + nlohmann::json(action->Env()).dump() + "\n"; if (response->HasStdOut()) { if (has_both) { message += "Stdout:\n"; } message += PrefixLines(response->StdOut()); } if (response->HasStdErr()) { if (has_both) { message += "Stderr:\n"; } message += PrefixLines(response->StdErr()); } return message; }; logger.Emit((has_err or has_out) ? LogLevel::Info : LogLevel::Debug, std::move(build_message)); } void static PrintError( Logger const& logger, gsl::not_null const& action, gsl::not_null const& progress) { std::ostringstream msg{}; if (action->Content().IsTreeOverlayAction() or action->Content().IsTreeAction()) { msg << "Failed to execute tree"; if (action->Content().IsTreeOverlayAction()) { msg << "-overlay"; } msg << " action."; } else { msg << "Failed to execute command "; msg << nlohmann::json(action->Command()).dump(); msg << " in environment "; msg << nlohmann::json(action->Env()).dump(); } auto const& origin_map = progress->OriginMap(); auto const origins = origin_map.find(action->Content().Id()); if (origins != origin_map.end() and not origins->second.empty()) { msg << "\nrequested by"; for (auto const& origin : origins->second) { msg << "\n - "; msg << origin.first.ToShortString( Evaluator::GetExpressionLogLimit()); msg << "#"; msg << origin.second; } } if (action->Content().IsTreeOverlayAction()) { msg << "\ninputs were"; std::vector inputs = action->Dependencies(); std::sort(inputs.begin(), inputs.end(), [](auto a, auto b) { return a.path < b.path; }); for (auto const& input : inputs) { msg << "\n - " << input.node->Content().Info()->digest.hash() << ":" << input.node->Content().Info()->digest.size() << ":" << ToChar(input.node->Content().Info()->type); } } logger.Emit(LogLevel::Error, "{}", msg.str()); } [[nodiscard]] static auto ScaleTime(std::chrono::milliseconds t, double f) -> std::chrono::milliseconds { return std::chrono::milliseconds( std::lround(static_cast(t.count()) * f)); } [[nodiscard]] static auto MergeProperties( const ExecutionProperties& base, const ExecutionProperties& overlay) { ExecutionProperties result = base; for (auto const& [k, v] : overlay) { result[k] = v; } return result; } private: /// \brief Get the alternative endpoint based on a specified set of platform /// properties. These are checked against the dispatch list of an existing /// remote context. [[nodiscard]] static auto GetAlternativeEndpoint( const ExecutionProperties& properties, const gsl::not_null& remote_context, HashFunction::Type hash_type, TmpDir::Ptr temp_space) -> std::unique_ptr { for (auto const& [pred, endpoint] : remote_context->exec_config->dispatch) { bool match = true; for (auto const& [k, v] : pred) { auto const v_it = properties.find(k); if (not(v_it != properties.end() and v_it->second == v)) { match = false; } } if (match) { Logger::Log(LogLevel::Debug, [endpoint = endpoint] { return fmt::format("Dispatching action to endpoint {}", endpoint.ToJson().dump()); }); ExecutionConfiguration config; return std::make_unique( "alternative remote execution", endpoint.host, endpoint.port, remote_context->auth, remote_context->retry_config, config, HashFunction{hash_type}, std::move(temp_space)); } } return nullptr; } }; /// \brief Executor for using concrete Execution API. class Executor { using Impl = ExecutorImpl; using CF = IExecutionAction::CacheFlag; public: /// \brief Create rebuilder for action comparision of two endpoints. /// \param context Execution context. References all the required /// information needed to execute actions on a specified remote endpoint. /// \param logger Overwrite the default logger. Useful for orchestrated /// builds, i.e., triggered by just serve. /// \param timeout Timeout for action execution. explicit Executor( gsl::not_null const& context, Logger const* logger = nullptr, // log in caller logger, if given std::chrono::milliseconds timeout = IExecutionAction::kDefaultTimeout) : context_{*context}, logger_{logger}, timeout_{timeout} {} /// \brief Run an action in a blocking manner /// This method must be thread-safe as it could be called in parallel /// \param[in] action The action to execute. /// \returns True if execution was successful, false otherwise [[nodiscard]] auto Process( gsl::not_null const& action) const noexcept -> bool { try { // to avoid always creating a logger we might not need, which is a // non-copyable and non-movable object, we need some code // duplication if (logger_ != nullptr) { auto const response = Impl::ExecuteAction( *logger_, action, *context_.apis->remote, Impl::MergeProperties(context_.remote_context->exec_config ->platform_properties, action->ExecutionProperties()), context_.remote_context, Impl::ScaleTime(timeout_, action->TimeoutScale()), action->NoCache() ? CF::DoNotCacheOutput : CF::CacheOutput, context_.statistics, context_.progress); // check response and save digests of results if (not response) { return true; } auto result = Impl::ParseResponse(*logger_, *response, action, context_.statistics, context_.progress); if (context_.profile) { (*context_.profile) ->NoteActionCompleted(action->Content().Id(), *response, action->Content().Cwd()); } return result; } Logger logger("action:" + action->Content().Id()); auto const response = Impl::ExecuteAction( logger, action, *context_.apis->remote, Impl::MergeProperties( context_.remote_context->exec_config->platform_properties, action->ExecutionProperties()), context_.remote_context, Impl::ScaleTime(timeout_, action->TimeoutScale()), action->NoCache() ? CF::DoNotCacheOutput : CF::CacheOutput, context_.statistics, context_.progress); // check response and save digests of results if (not response) { return true; } auto result = Impl::ParseResponse(logger, *response, action, context_.statistics, context_.progress); if (context_.profile) { (*context_.profile) ->NoteActionCompleted(action->Content().Id(), *response, action->Content().Cwd()); } return result; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Executor: Unexpected failure processing action with:\n{}", ex.what()); return false; } } /// \brief Check artifact is available to the CAS or upload it. /// \param[in] artifact The artifact to process. /// \param[in] repo_config The repository configuration to use /// \returns True if artifact is available or uploaded, false otherwise [[nodiscard]] auto Process( gsl::not_null const& artifact) const noexcept -> bool { try { // to avoid always creating a logger we might not need, which is a // non-copyable and non-movable object, we need some code // duplication if (logger_ != nullptr) { return Impl::VerifyOrUploadArtifact( *logger_, artifact, context_.repo_config, *context_.apis); } Logger logger("artifact:" + ToHexString(artifact->Content().Id())); return Impl::VerifyOrUploadArtifact( logger, artifact, context_.repo_config, *context_.apis); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Executor: Unexpected failure checking artifact with:\n{}", ex.what()); return false; } } private: ExecutionContext const& context_; Logger const* logger_; std::chrono::milliseconds timeout_; }; /// \brief Rebuilder for running and comparing actions of two API endpoints. class Rebuilder { using Impl = ExecutorImpl; using CF = IExecutionAction::CacheFlag; public: /// \brief Create rebuilder for action comparision of two endpoints. /// \param context Execution context. References all the required /// information needed to perform a rebuild, during which the results of /// executing actions on the regular remote endpoint and the cache endpoint /// are compared. /// \param timeout Timeout for action execution. explicit Rebuilder( gsl::not_null const& context, std::chrono::milliseconds timeout = IExecutionAction::kDefaultTimeout) : context_{*context}, api_cached_{context_.apis->MakeRemote( context_.remote_context->exec_config->cache_address, context_.remote_context->auth, context_.remote_context->retry_config)}, timeout_{timeout} {} [[nodiscard]] auto Process( gsl::not_null const& action) const noexcept -> bool { try { auto const& action_id = action->Content().Id(); Logger logger("rebuild:" + action_id); auto response = Impl::ExecuteAction( logger, action, *context_.apis->remote, Impl::MergeProperties( context_.remote_context->exec_config->platform_properties, action->ExecutionProperties()), context_.remote_context, Impl::ScaleTime(timeout_, action->TimeoutScale()), CF::PretendCached, context_.statistics, context_.progress); if (not response) { return true; // action without response (e.g., tree action) } Logger logger_cached("cached:" + action_id); auto response_cached = Impl::ExecuteAction( logger_cached, action, *api_cached_, Impl::MergeProperties( context_.remote_context->exec_config->platform_properties, action->ExecutionProperties()), context_.remote_context, Impl::ScaleTime(timeout_, action->TimeoutScale()), CF::FromCacheOnly, context_.statistics, context_.progress); if (not response_cached) { logger_cached.Emit(LogLevel::Error, "expected regular action with response"); return false; } if (auto error = DetectFlakyAction( *response, *response_cached, action->Content())) { logger_cached.Emit(LogLevel::Error, *error); return false; } return Impl::ParseResponse(logger, *response, action, context_.statistics, context_.progress, /*count_as_executed=*/true); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Rebuilder: Unexpected failure processing action with:\n{}", ex.what()); return false; } } [[nodiscard]] auto Process( gsl::not_null const& artifact) const noexcept -> bool { try { Logger logger("artifact:" + ToHexString(artifact->Content().Id())); return Impl::VerifyOrUploadArtifact( logger, artifact, context_.repo_config, *context_.apis); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Rebuilder: Unexpected failure checking artifact with:\n{}", ex.what()); return false; } } [[nodiscard]] auto DumpFlakyActions() const -> nlohmann::json { std::unique_lock lock{m_}; auto actions = nlohmann::json::object(); for (auto const& [action_id, outputs] : flaky_actions_) { for (auto const& [path, infos] : outputs) { actions[action_id][path]["rebuilt"] = infos.first.ToJson(); actions[action_id][path]["cached"] = infos.second.ToJson(); } } return {{"flaky actions", actions}, {"cache misses", cache_misses_}}; } private: ExecutionContext const& context_; gsl::not_null const api_cached_; std::chrono::milliseconds timeout_; mutable std::mutex m_; mutable std::vector cache_misses_; mutable std::unordered_map< std::string, std::unordered_map< std::string, std::pair>> flaky_actions_; [[nodiscard]] auto DetectFlakyAction( IExecutionResponse::Ptr const& response, IExecutionResponse::Ptr const& response_cached, Action const& action) const -> std::optional { auto& stats = *context_.statistics; if (response and response_cached and response_cached->ActionDigest() == response->ActionDigest()) { stats.IncrementRebuiltActionComparedCounter(); auto const artifacts = response->Artifacts(); if (not artifacts) { return artifacts.error(); } auto const artifacts_cached = response_cached->Artifacts(); if (not artifacts_cached) { return artifacts_cached.error(); } std::ostringstream msg{}; try { for (auto const& [path, info] : *artifacts.value()) { auto const& info_cached = artifacts_cached.value()->at(path); if (info != info_cached) { RecordFlakyAction( &msg, action, path, info, info_cached); } } } catch (std::exception const& ex) { return ex.what(); } if (msg.tellp() > 0) { stats.IncrementActionsFlakyCounter(); bool tainted = action.MayFail() or action.NoCache(); if (tainted) { stats.IncrementActionsFlakyTaintedCounter(); } Logger::Log(tainted ? LogLevel::Debug : LogLevel::Warning, "{}", msg.str()); } } else { stats.IncrementRebuiltActionMissingCounter(); std::unique_lock lock{m_}; cache_misses_.emplace_back(action.Id()); } return std::nullopt; // ok } void RecordFlakyAction(gsl::not_null const& msg, Action const& action, std::string const& path, Artifact::ObjectInfo const& rebuilt, Artifact::ObjectInfo const& cached) const noexcept { auto const& action_id = action.Id(); if (msg->tellp() <= 0) { bool tainted = action.MayFail() or action.NoCache(); auto cmd = GetCmdString(action); (*msg) << "Found flaky " << (tainted ? "tainted " : "") << "action:" << std::endl << " - id: " << action_id << std::endl << " - cmd: " << cmd << std::endl; } (*msg) << " - output '" << path << "' differs:" << std::endl << " - " << rebuilt.ToString() << " (rebuilt)" << std::endl << " - " << cached.ToString() << " (cached)" << std::endl; std::unique_lock lock{m_}; auto& object_map = flaky_actions_[action_id]; try { object_map.emplace(path, std::make_pair(rebuilt, cached)); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "recoding flaky action failed with: {}", ex.what()); } } static auto GetCmdString(Action const& action) noexcept -> std::string { try { return nlohmann::json(action.Command()).dump(); } catch (std::exception const& ex) { return fmt::format("", ex.what()); } } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/traverser/000077500000000000000000000000001516554100600277505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/traverser/TARGETS000066400000000000000000000007271516554100600310120ustar00rootroot00000000000000{ "traverser": { "type": ["@", "rules", "CC", "library"] , "name": ["traverser"] , "hdrs": ["traverser.hpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] ] , "stage": ["src", "buildtool", "execution_engine", "traverser"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/traverser/traverser.hpp000066400000000000000000000173741516554100600325120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" /// \brief Concept required for Runners used by the Traverser. template concept Runnable = requires(T const r, DependencyGraph::ActionNode const* action, DependencyGraph::ArtifactNode const* artifact) { { r.Process(action) } -> std::same_as; { r.Process(artifact) } -> std::same_as; }; /// \brief Class to traverse the dependency graph executing necessary actions /// \tparam Executor Type of the executor // Traversal of the graph and execution of actions are concurrent, using /// the //src/buildtool/execution_engine/task_system. /// Graph remains constant and the only parts of the nodes that are modified are /// their traversal state template class Traverser { public: explicit Traverser(Executor const& r, DependencyGraph const& graph, std::size_t jobs, gsl::not_null*> const& fail_flag) : runner_{r}, graph_{graph}, failed_{fail_flag}, tasker_{jobs} {} Traverser() = delete; Traverser(Traverser const&) = delete; Traverser(Traverser&&) = delete; auto operator=(Traverser const&) -> Traverser& = delete; auto operator=(Traverser&&) -> Traverser& = delete; ~Traverser() = default; // Traverse the whole graph [[nodiscard]] auto Traverse() noexcept -> bool { auto const& ids = graph_.ArtifactIdentifiers(); return Traverse(ids); }; // Traverse starting by the artifacts with the given identifiers, avoiding // executing actions that are not strictly needed to build the given // artifacts [[nodiscard]] auto Traverse(std::unordered_set const& target_ids) noexcept -> bool; private: Executor const& runner_{}; DependencyGraph const& graph_; gsl::not_null*> failed_; TaskSystem tasker_; // THIS SHOULD BE THE LAST MEMBER VARIABLE // Visits discover nodes and queue visits to their children nodes. void Visit(gsl::not_null artifact_node) noexcept; void Visit( gsl::not_null action_node) noexcept; // Notify all actions that have it as a dependency that it is available and // queue execution of those that become ready (that were only waiting for // this artifact) void NotifyAvailable( gsl::not_null const& artifact_node) noexcept; // Calls NotifyAvailable on all the action's outputs void NotifyAvailable( gsl::not_null const& action_node) noexcept; // Visit to nodes are queued only once template void QueueVisit(NodeTypePtr node) noexcept { // in case the node was already discovered, there is no need to queue // the visit if (failed_->load() or node->TraversalState()->GetAndMarkDiscovered()) { return; } tasker_.QueueTask([this, node]() noexcept { Visit(node); }); } // Queue task to process the node by the executor after making sure that the // node is required and that it was not yet queued to be processed. The task // queued will call notify that the node is available in case processing it // was successful template void QueueProcessing(NodeTypePtr node) noexcept { if (failed_->load() or not node->TraversalState()->IsRequired() or node->TraversalState()->GetAndMarkQueuedToBeProcessed()) { return; } auto process_node = [this, node]() { if (runner_.Process(node)) { NotifyAvailable(node); } else { Abort(); } }; tasker_.QueueTask(process_node); } void Abort() noexcept { failed_->store(true); tasker_.Shutdown(); // skip execution of pending tasks } }; template auto Traverser::Traverse( std::unordered_set const& target_ids) noexcept -> bool { for (auto artifact_id : target_ids) { auto const* artifact_node = graph_.ArtifactNodeWithId(artifact_id); if (artifact_node != nullptr) { QueueVisit(artifact_node); } else { Abort(); Logger::Log( LogLevel::Error, "artifact with id {} can not be found in dependency graph.", artifact_id); return false; } } return true; } template void Traverser::Visit( gsl::not_null artifact_node) noexcept { artifact_node->TraversalState()->MarkRequired(); // Visits are queued only once per artifact node, but it could be that the // builder action had multiple outputs and was queued and executed through // the visit to another of the outputs, in which case the current artifact // would be available and there is nothing else to do if (artifact_node->TraversalState()->IsAvailable()) { return; } if (artifact_node->HasBuilderAction()) { QueueVisit(gsl::not_null( artifact_node->BuilderActionNode())); } else { QueueProcessing(artifact_node); } } template void Traverser::Visit( gsl::not_null action_node) noexcept { action_node->TraversalState()->MarkRequired(); for (auto const& dep : action_node->Children()) { if (not dep->TraversalState()->IsAvailable()) { QueueVisit(dep); } } if (action_node->TraversalState()->IsReady()) { QueueProcessing(action_node); } } template void Traverser::NotifyAvailable( gsl::not_null const& artifact_node) noexcept { artifact_node->TraversalState()->MakeAvailable(); for (auto const& action_node : artifact_node->Parents()) { if (action_node->TraversalState()->NotifyAvailableDepAndCheckReady()) { QueueProcessing(action_node); } } } template void Traverser::NotifyAvailable( gsl::not_null const& action_node) noexcept { for (auto const& output : action_node->Parents()) { NotifyAvailable(output); } } #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/tree_operations/000077500000000000000000000000001516554100600311355ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/tree_operations/TARGETS000066400000000000000000000021431516554100600321710ustar00rootroot00000000000000{ "tree_operations_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["tree_operations_utils"] , "hdrs": ["tree_operations_utils.hpp"] , "srcs": ["tree_operations_utils.cpp"] , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hash_combine"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "execution_engine", "tree_operations"] } } tree_operations_utils.cpp000066400000000000000000000412231516554100600362060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/tree_operations// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_engine/tree_operations/tree_operations_utils.hpp" #include #include #include #include #include #include #include // for remove_reference #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/utils/cpp/hex_string.hpp" auto TreeOperationsUtils::ParseBazelDirectory( std::string const& tree_data, HashFunction::Type hash_type) noexcept -> std::optional { bazel_re::Directory bazel_directory{}; if (not bazel_directory.ParseFromString(tree_data)) { return std::nullopt; } // Collect all entries from bazel directory. TreeEntries tree_entries{}; tree_entries.reserve( static_cast(bazel_directory.files_size()) + static_cast(bazel_directory.symlinks_size()) + static_cast(bazel_directory.directories_size())); // Collect files. for (auto const& file : bazel_directory.files()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, file.digest()); if (not digest) { return std::nullopt; } tree_entries.insert( {file.name(), TreeEntry{.info = Artifact::ObjectInfo{ .digest = *std::move(digest), .type = file.is_executable() ? ObjectType::Executable : ObjectType::File}}}); } // Collect symlinks. HashFunction hash_function{hash_type}; for (auto const& symlink : bazel_directory.symlinks()) { tree_entries.insert( {symlink.name(), TreeEntry{ .info = Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs< ObjectType::File>(hash_function, symlink.target()), .type = ObjectType::Symlink}, .symlink_target = symlink.target()}}); } // Collect directories. for (auto const& dir : bazel_directory.directories()) { auto digest = ArtifactDigestFactory::FromBazel(hash_type, dir.digest()); if (not digest) { return std::nullopt; } tree_entries.insert({dir.name(), TreeEntry{.info = Artifact::ObjectInfo{ .digest = *std::move(digest), .type = ObjectType::Tree}}}); } return tree_entries; } auto TreeOperationsUtils::ParseGitTree(std::string const& tree_data, ArtifactDigest const& tree_digest, HashFunction::Type hash_type) noexcept -> std::optional { // For a tree-overlay computation, the actual target of a symbolic // link is not relevant. Symbolic links are just considered as // regular blobs. auto git_entries = GitRepo::ReadTreeData( tree_data, tree_digest.hash(), [](std::vector const&) { return true; }, /*is_hex_id=*/true); if (not git_entries) { return std::nullopt; } // Collect all entries from git tree. TreeEntries tree_entries{}; tree_entries.reserve(git_entries->size()); for (auto const& [git_hash, entries] : *git_entries) { // Pick the first entry for that git hash to calculate the // object info once, since all follow-up entries will be the // same object, just with a different name. auto const& first_entry = entries.front(); auto digest = ArtifactDigestFactory::Create(hash_type, ToHexString(git_hash), /*size=*/0, IsTreeObject(first_entry.type)); if (not digest) { return std::nullopt; } // Pick up all names for that git object and create a tree entry // for each of them. for (auto const& entry : entries) { tree_entries.insert( {entry.name, TreeEntry{.info = Artifact::ObjectInfo{ .digest = *digest, .type = first_entry.type}}}); } } return tree_entries; } auto TreeOperationsUtils::ReadTree( IExecutionApi const& api, Artifact::ObjectInfo const& tree_info) noexcept -> expected { // Fetch tree data. auto tree_data = api.RetrieveToMemory(tree_info); if (not tree_data) { return unexpected{ fmt::format("Failed to fetch tree: {}", tree_info.ToString())}; } // Parse tree data. auto tree_entries = ProtocolTraits::IsNative(api.GetHashType()) ? ParseGitTree(*tree_data, tree_info.digest, api.GetHashType()) : ParseBazelDirectory(*tree_data, api.GetHashType()); if (not tree_entries) { return unexpected{ fmt::format("Failed to parse tree: {}", tree_info.ToString())}; } return *tree_entries; } auto TreeOperationsUtils::SerializeBazelDirectory( TreeEntries const& tree_entries) noexcept -> std::optional { // Sort tree entry names, so we can process them in the correct order. auto sorted = std::set{}; std::transform(tree_entries.begin(), tree_entries.end(), std::inserter(sorted, sorted.end()), [](auto const& name_entry) { return name_entry.first; }); // Convert tree entries to bazel directory. bazel_re::Directory bazel_directory{}; for (auto const& name : sorted) { auto const& entry = tree_entries.at(name); switch (entry.info.type) { case ObjectType::File: case ObjectType::Executable: { auto* file = bazel_directory.add_files(); file->set_name(name); *file->mutable_digest() = ArtifactDigestFactory::ToBazel(entry.info.digest); file->set_is_executable(entry.info.type == ObjectType::Executable); break; } case ObjectType::Symlink: { auto* symlink = bazel_directory.add_symlinks(); symlink->set_name(name); symlink->set_target(*entry.symlink_target); break; } case ObjectType::Tree: { auto* dir = bazel_directory.add_directories(); dir->set_name(name); *dir->mutable_digest() = ArtifactDigestFactory::ToBazel(entry.info.digest); break; } default: { return std::nullopt; } } } // Serialize bazel directory. return bazel_directory.SerializeAsString(); } auto TreeOperationsUtils::SerializeGitTree( TreeEntries const& tree_entries) noexcept -> std::optional { // Convert tree entries to git entries. GitRepo::tree_entries_t git_entries{}; git_entries.reserve(tree_entries.size()); for (auto const& [name, entry] : tree_entries) { auto git_hash = FromHexString(entry.info.digest.hash()); if (not git_hash) { return std::nullopt; } auto it = git_entries.find(*git_hash); if (it == git_entries.end()) { if (auto res = git_entries.insert( {*git_hash, std::vector{}}); res.second) { it = std::move(res).first; } else { return std::nullopt; } } it->second.emplace_back(name, entry.info.type); } // Serialize git entries. auto git_tree = GitRepo::CreateShallowTree(git_entries); if (not git_tree) { return std::nullopt; } return git_tree->second; } auto TreeOperationsUtils::WriteTree(IExecutionApi const& api, TreeEntries const& tree_entries) noexcept -> expected { // Serialize tree entries. auto tree_data = ProtocolTraits::IsNative(api.GetHashType()) ? SerializeGitTree(tree_entries) : SerializeBazelDirectory(tree_entries); if (not tree_data) { return unexpected{fmt::format("Failed to serialize tree entries")}; } // Write tree data. auto tree_blob = ArtifactBlob::FromMemory(HashFunction{api.GetHashType()}, ObjectType::Tree, *std::move(tree_data)); if (not tree_blob) { return unexpected{fmt::format("Failed to create tree blob")}; } if (api.Upload(std::unordered_set{*tree_blob})) { return Artifact::ObjectInfo{.digest = tree_blob->GetDigest(), .type = ObjectType::Tree}; } return unexpected{fmt::format("Failed to upload tree blob")}; } auto TreeOperationsUtils::CreateTreeOverlayMap(IExecutionApi const& api, bool disjoint) noexcept -> AsyncMapConsumer { auto value_creator = [&api, disjoint](auto /*unused*/, auto const& setter, auto const& logger, auto const& subcaller, auto const& key) { auto const& base_tree_info = key.trees.first; auto const& other_tree_info = key.trees.second; Logger::Log(LogLevel::Trace, "Compute tree overlay:\n - {}\n - {}", base_tree_info.ToString(), other_tree_info.ToString()); // Wrap logger for this tree-overlay computation. auto new_logger = std::make_shared( [logger, base_tree_info, other_tree_info](auto const& msg, auto fatal) { (*logger)(fmt::format("While merging the trees:\n - {}\n " "- {}\n{}", base_tree_info.ToString(), other_tree_info.ToString(), msg), fatal); }); // Ensure that both objects are actually trees. if (not IsTreeObject(base_tree_info.type) or not IsTreeObject(other_tree_info.type)) { (*new_logger)(fmt::format("Both objects have to be trees."), /*fatal=*/true); return; } // Early return if both trees are the same. if (base_tree_info == other_tree_info) { (*setter)(Artifact::ObjectInfo{base_tree_info}); return; } // Read base tree. auto base_tree = ReadTree(api, base_tree_info); if (not base_tree) { (*new_logger)(base_tree.error(), /*fatal=*/true); return; } // Read other tree. auto other_tree = ReadTree(api, other_tree_info); if (not other_tree) { (*new_logger)(other_tree.error(), /*fatal=*/true); return; } // Compute tree overlay. If two trees conflict, collect them and // process them in the subcaller. TreeEntries overlay_tree{*other_tree}; // Make a copy of other tree. std::vector keys{}; std::vector base_names{}; auto min_size = std::min(base_tree->size(), other_tree->size()); keys.reserve(min_size); base_names.reserve(min_size); for (auto& [base_name, base_entry] : *base_tree) { auto it = overlay_tree.find(base_name); if (it == overlay_tree.end()) { // No naming conflict detected, add entry to the other // tree. overlay_tree[std::move(base_name)] = std::move(base_entry); continue; } if (it->second.info == base_entry.info) { // Naming conflict detected, but names point to the same // object, no conflict. continue; } // Naming conflict detected and names point to different // objects. if (IsTreeObject(base_entry.info.type) and IsTreeObject(it->second.info.type)) { // If both objects are trees, compute tree overlay in // the subcaller. keys.emplace_back(std::make_pair(std::move(base_entry.info), std::move(it->second.info))); base_names.emplace_back(std::move(base_name)); continue; } // If both objects are not trees, actual conflict detected. if (disjoint) { (*new_logger)(fmt::format("Naming conflict detected at path " "{}:\n - {}\n - {}", nlohmann::json(base_name).dump(), base_entry.info.ToString(), it->second.info.ToString()), /*fatal=*/true); return; } // Ignore conflict, keep entry from other tree. } (*subcaller)( keys, [setter, new_logger, &api, base_names = std::move(base_names), partial_overlay_tree = std::move(overlay_tree)](auto const& values) { // Insert computed tree overlays into tree-overlay // entries. TreeEntries overlay_tree{partial_overlay_tree}; for (size_t i = 0; i < values.size(); ++i) { // Create copy of the value. overlay_tree[base_names[i]] = TreeEntry{.info = *(values[i])}; } // Write tree overlay. auto overlay_tree_info = WriteTree(api, overlay_tree); if (not overlay_tree_info) { (*new_logger)(overlay_tree_info.error(), /*fatal=*/true); return; } Logger::Log(LogLevel::Trace, "Tree-overlay result: {}", overlay_tree_info->ToString()); (*setter)(*std::move(overlay_tree_info)); }, new_logger); }; return AsyncMapConsumer{value_creator}; } auto TreeOperationsUtils::ComputeTreeOverlay( IExecutionApi const& api, Artifact::ObjectInfo const& base_tree_info, Artifact::ObjectInfo const& other_tree_info, bool disjoint) noexcept -> expected { auto tree_overlay_map = CreateTreeOverlayMap(api, disjoint); Artifact::ObjectInfo result{}; std::string failed_msg{}; bool failed{false}; { TaskSystem ts{1}; tree_overlay_map.ConsumeAfterKeysReady( &ts, {TreePair{std::make_pair(base_tree_info, other_tree_info)}}, [&result](auto const& values) { result = *values[0]; }, [&failed_msg, &failed](auto const& msg, auto fatal) { failed_msg = msg; failed = fatal; }); } if (failed) { return unexpected{failed_msg}; } return result; } tree_operations_utils.hpp000066400000000000000000000116601516554100600362150ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/execution_engine/tree_operations// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TREE_OPERATIONS_TREE_OPERATIONS_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TREE_OPERATIONS_TREE_OPERATIONS_UTILS_HPP #include #include #include #include #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hash_combine.hpp" namespace std { template struct hash; } /// \brief Utility functions for tree operations. class TreeOperationsUtils final { public: struct TreeEntry { Artifact::ObjectInfo info{}; std::optional symlink_target{}; }; using TreeEntries = std::unordered_map; /// \brief Computes a new tree from two existing ones by overlaying /// their contents. /// \param api The execution API to be used. /// \param base_tree_info The base tree to be overlayed with another tree. /// \param other_tree_info The other tree to be overlayed with the /// base tree. /// \param disjoint If true, abort the computation if a /// conflict is encountered, otherwise the conflict is ignored and /// the entry from the second tree wins. /// \returns The computed overlayed tree or an error message in case /// of a conflict, when disjoint mode is used. [[nodiscard]] static auto ComputeTreeOverlay( IExecutionApi const& api, Artifact::ObjectInfo const& base_tree_info, Artifact::ObjectInfo const& other_tree_info, bool disjoint) noexcept -> expected; [[nodiscard]] static auto ReadTree( IExecutionApi const& api, Artifact::ObjectInfo const& tree_info) noexcept -> expected; [[nodiscard]] static auto WriteTree( IExecutionApi const& api, TreeEntries const& tree_entries) noexcept -> expected; private: struct TreePair { std::pair trees; explicit TreePair( std::pair trees) : trees{std::move(trees)} {} [[nodiscard]] auto operator==(TreePair const& other) const noexcept -> bool { return trees == other.trees; } }; friend struct std::hash; [[nodiscard]] static auto ParseBazelDirectory( std::string const& tree_data, HashFunction::Type hash_type) noexcept -> std::optional; [[nodiscard]] static auto ParseGitTree( std::string const& tree_data, ArtifactDigest const& tree_digest, HashFunction::Type hash_type) noexcept -> std::optional; [[nodiscard]] static auto SerializeBazelDirectory( TreeEntries const& tree_entries) noexcept -> std::optional; [[nodiscard]] static auto SerializeGitTree( TreeEntries const& tree_entries) noexcept -> std::optional; /// \brief Creates an async map consumer that maps a pair of trees /// to their corresponding overlay tree. /// \param api The execution API to be used. /// \param disjoint If true, abort the computation if a /// conflict is encountered, otherwise the conflict is ignored and /// the entry from the second tree wins. /// \returns The async map consumer. [[nodiscard]] static auto CreateTreeOverlayMap(IExecutionApi const& api, bool disjoint) noexcept -> AsyncMapConsumer; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(TreeOperationsUtils::TreePair const& pair) const noexcept -> std::size_t { std::size_t seed{}; hash_combine(&seed, pair.trees.first); hash_combine(&seed, pair.trees.second); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TREE_OPERATIONS_TREE_OPERATIONS_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/000077500000000000000000000000001516554100600247265ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/TARGETS000066400000000000000000000156571516554100600260000ustar00rootroot00000000000000{ "object_type": { "type": ["@", "rules", "CC", "library"] , "name": ["object_type"] , "hdrs": ["object_type.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "file_system"] } , "file_storage": { "type": ["@", "rules", "CC", "library"] , "name": ["file_storage"] , "hdrs": ["file_storage.hpp"] , "deps": [ "object_type" , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "file_system"] } , "object_cas": { "type": ["@", "rules", "CC", "library"] , "name": ["object_cas"] , "hdrs": ["object_cas.hpp"] , "deps": [ "file_storage" , "object_type" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "file_system"] } , "file_system_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["file_system_manager"] , "hdrs": ["file_system_manager.hpp"] , "deps": [ "object_type" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/system", "system"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] , ["src/utils/cpp", "path"] ] , "stage": ["src", "buildtool", "file_system"] } , "jsonfs": { "type": ["@", "rules", "CC", "library"] , "name": ["jsonfs"] , "hdrs": ["jsonfs.hpp"] , "deps": [ "file_system_manager" , "object_type" , ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "file_system"] } , "git_cas": { "type": ["@", "rules", "CC", "library"] , "name": ["git_cas"] , "hdrs": ["git_cas.hpp"] , "srcs": ["git_cas.cpp"] , "deps": [ "git_utils" , "object_type" , ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "log_level"] ] , "stage": ["src", "buildtool", "file_system"] , "private-deps": [ "git_context" , ["", "libgit2"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path"] ] } , "git_tree": { "type": ["@", "rules", "CC", "library"] , "name": ["git_tree"] , "hdrs": ["git_tree.hpp"] , "srcs": ["git_tree.cpp"] , "deps": [ "git_cas" , "git_repo" , "object_type" , ["@", "gsl", "", "gsl"] , ["src/buildtool/multithreading", "atomic_value"] , ["src/utils/cpp", "hex_string"] ] , "stage": ["src", "buildtool", "file_system"] , "private-deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "git_context": { "type": ["@", "rules", "CC", "library"] , "name": ["git_context"] , "hdrs": ["git_context.hpp"] , "srcs": ["git_context.cpp"] , "stage": ["src", "buildtool", "file_system"] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["", "libgit2"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "git_repo": { "type": ["@", "rules", "CC", "library"] , "name": ["git_repo"] , "hdrs": ["git_repo.hpp"] , "srcs": ["git_repo.cpp"] , "deps": [ "git_cas" , "git_types" , "git_utils" , "object_type" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "file_system"] , "private-deps": [ "git_context" , ["@", "fmt", "", "fmt"] , ["", "libgit2"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "file_locking"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "tmp_dir"] ] , "cflags": ["-pthread"] } , "git_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["git_utils"] , "hdrs": ["git_utils.hpp"] , "srcs": ["git_utils.cpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "file_system"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["", "libgit2"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "hex_string"] ] } , "git_types": { "type": ["@", "rules", "CC", "library"] , "name": ["git_types"] , "hdrs": ["git_types.hpp"] , "stage": ["src", "buildtool", "file_system"] } , "file_root": { "type": ["@", "rules", "CC", "library"] , "name": ["file_root"] , "hdrs": ["file_root.hpp"] , "deps": [ "file_system_manager" , "git_cas" , "git_tree" , "git_tree_utils" , "object_type" , "precomputed_root" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "git_hashes_converter"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "concepts"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "json"] ] , "stage": ["src", "buildtool", "file_system"] } , "precomputed_root": { "type": ["@", "rules", "CC", "library"] , "name": ["precomputed_root"] , "hdrs": ["precomputed_root.hpp"] , "srcs": ["precomputed_root.cpp"] , "deps": [ ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "buildtool", "file_system"] } , "atomic": { "type": ["@", "rules", "CC", "library"] , "name": ["atomic"] , "hdrs": ["atomic.hpp"] , "srcs": ["atomic.cpp"] , "private-deps": [["@", "fmt", "", "fmt"]] , "stage": ["src", "buildtool", "file_system"] } , "git_tree_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["git_tree_utils"] , "hdrs": ["git_tree_utils.hpp"] , "srcs": ["git_tree_utils.cpp"] , "deps": ["git_cas", "git_tree", ["src/buildtool/storage", "config"]] , "private-deps": [ "file_system_manager" , "object_type" , ["@", "gsl", "", "gsl"] , ["src/buildtool/storage", "fs_utils"] ] , "stage": ["src", "buildtool", "file_system"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/atomic.cpp000066400000000000000000000040441516554100600267100ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/atomic.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include "fmt/core.h" auto FileSystemAtomic::WriteFile(std::string const& filename, std::string const& content) noexcept -> bool { // As there is no high-level replacement of mkstemp(3), we fall back to // using the libc functions. auto filename_tmp_template = fmt::format("{}.XXXXXX", filename); char* filename_tmp = strdup(filename_tmp_template.c_str()); if (filename_tmp == nullptr) { return false; } auto fd = mkstemp(filename_tmp); if (fd == -1) { free(filename_tmp); // NOLINT return false; } size_t to_write = content.length(); const char* buf = content.c_str(); while (to_write > 0) { auto written = write(fd, buf, to_write); if (written < 0) { (void)close(fd); free(filename_tmp); // NOLINT return false; } to_write -= static_cast(written); buf = &buf[written]; // NOLINT } if (close(fd) != 0) { free(filename_tmp); // NOLINT return false; } if (rename(filename_tmp, filename.c_str()) != 0) { free(filename_tmp); // NOLINT return false; } free(filename_tmp); // NOLINT return true; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/atomic.hpp000066400000000000000000000021221516554100600267100ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_ATOMIC_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_ATOMIC_HPP #include class FileSystemAtomic { public: /// \brief Write file atomically /// First write the contents to a temporary file in the same directory; only /// once completed, rename the fie to the final destination. static auto WriteFile(std::string const& filename, std::string const& content) noexcept -> bool; }; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/file_root.hpp000066400000000000000000001145251516554100600274310ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/git_hashes_converter.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/buildtool/file_system/git_tree_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/concepts.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" // Keep it to ensure fmt::format works on JSON objects #include "src/utils/cpp/json.hpp" // IWYU pragma: keep /// FilteredIterator is an helper class to allow for iteration over /// directory-only or file-only entries stored inside the class /// DirectoryEntries. Internally, DirectoryEntries holds a /// map or map. While iterating, we are /// just interested in the name of the entry (i.e., the string). /// To decide which entries retain, the FilteredIterator requires a predicate. template // I is a forward iterator // I::value_type is a std::pair class FilteredIterator { public: using value_type = std::string const; using pointer = value_type*; using reference = value_type&; using difference_type = std::ptrdiff_t; using iteratori_category = std::forward_iterator_tag; using predicate_t = std::function; FilteredIterator() noexcept = default; // [first, last) is a valid sequence FilteredIterator(I first, I last, predicate_t p) noexcept : iterator_{std::find_if(first, last, p)}, end_{std::move(last)}, p_{std::move(p)} {} auto operator*() const noexcept -> reference { return iterator_->first; } auto operator++() noexcept -> FilteredIterator& { ++iterator_; iterator_ = std::find_if(iterator_, end_, p_); return *this; } [[nodiscard]] auto begin() noexcept -> FilteredIterator& { return *this; } [[nodiscard]] auto end() const noexcept -> FilteredIterator { return FilteredIterator{end_, end_, p_}; } [[nodiscard]] friend auto operator==(FilteredIterator const& x, FilteredIterator const& y) noexcept -> bool { return x.iterator_ == y.iterator_; } [[nodiscard]] friend auto operator!=(FilteredIterator const& x, FilteredIterator const& y) noexcept -> bool { return not(x == y); } private: I iterator_{}; const I end_{}; predicate_t p_{}; }; class FileRoot { using fs_root_t = std::filesystem::path; struct RootGit { gsl::not_null cas; gsl::not_null tree; gsl::not_null root_entry; }; // absent roots are defined by a tree hash with no witnessing repository using absent_root_t = std::string; using root_t = std::variant; public: static constexpr auto kGitTreeMarker = "git tree"; static constexpr auto kGitTreeIgnoreSpecialMarker = "git tree ignore-special"; static constexpr auto kFileIgnoreSpecialMarker = "file ignore-special"; static constexpr auto kComputedMarker = "computed"; class DirectoryEntries { friend class FileRoot; public: using pairs_t = std::unordered_map; using tree_t = gsl::not_null; using entries_t = std::variant; using fs_iterator_type = typename pairs_t::const_iterator; using fs_iterator = FilteredIterator; using git_iterator_type = GitTree::entries_t::const_iterator; using git_iterator = FilteredIterator; private: /// Iterator has two FilteredIterators, one for iterating over pairs_t /// and one for tree_t. Each FilteredIterator is constructed with a /// proper predicate, allowing for iteration on file-only or /// directory-only entries class Iterator { public: using value_type = std::string const; using pointer = value_type*; using reference = value_type&; using difference_type = std::ptrdiff_t; using iteratori_category = std::forward_iterator_tag; explicit Iterator(fs_iterator fs_it) : it_{std::move(fs_it)} {} explicit Iterator(git_iterator git_it) : it_{std::move(git_it)} {} auto operator*() const noexcept -> reference { if (std::holds_alternative(it_)) { return *std::get(it_); } return *std::get(it_); } [[nodiscard]] auto begin() noexcept -> Iterator& { return *this; } [[nodiscard]] auto end() const noexcept -> Iterator { if (std::holds_alternative(it_)) { return Iterator{std::get(it_).end()}; } return Iterator{std::get(it_).end()}; } auto operator++() noexcept -> Iterator& { if (std::holds_alternative(it_)) { ++(std::get(it_)); } else { ++(std::get(it_)); } return *this; } friend auto operator==(Iterator const& x, Iterator const& y) noexcept -> bool { try { return x.it_ == y.it_; } catch (std::exception const& e) { try { Logger::Log(LogLevel::Error, "Unexpected exception: {}", e.what()); std::terminate(); } catch (...) { std::terminate(); } } catch (...) { std::terminate(); } } friend auto operator!=(Iterator const& x, Iterator const& y) noexcept -> bool { return not(x == y); } private: std::variant it_; }; public: explicit DirectoryEntries(pairs_t pairs) noexcept : data_{std::move(pairs)} {} explicit DirectoryEntries(tree_t const& git_tree) noexcept : data_{git_tree} {} [[nodiscard]] auto ContainsBlob(std::string const& name) const noexcept -> bool { if (auto const* const data = std::get_if(&data_)) { auto const ptr = (*data)->LookupEntryByName(name); return ptr != nullptr and IsBlobObject(ptr->Type()); } if (auto const* const data = std::get_if(&data_)) { auto const it = data->find(name); return it != data->end() and IsBlobObject(it->second); } return false; } [[nodiscard]] auto Empty() const noexcept -> bool { if (std::holds_alternative(data_)) { try { auto const& tree = std::get(data_); return tree->begin() == tree->end(); } catch (...) { return false; } } if (std::holds_alternative(data_)) { return std::get(data_).empty(); } return true; } /// \brief Retrieve a root tree as a KNOWN artifact. /// Only succeeds if no entries have to be ignored. [[nodiscard]] auto AsKnownTree(HashFunction::Type hash_type, std::string const& repository) const noexcept -> std::optional { if (not ProtocolTraits::IsNative(hash_type)) { return std::nullopt; } if (std::holds_alternative(data_)) { try { auto const& data = std::get(data_); // only consider tree if we have it unmodified if (auto id = data->Hash()) { auto const& size = data->Size(); if (size) { auto digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, *id, *size, /*is_tree=*/true); if (not digest) { return std::nullopt; } return ArtifactDescription::CreateKnown( *std::move(digest), ObjectType::Tree, repository); } } } catch (...) { return std::nullopt; } } return std::nullopt; } [[nodiscard]] auto FilesIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsFileObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return IsFileObject(x.second->Type()); }}}; } [[nodiscard]] auto SymlinksIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsSymlinkObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return IsSymlinkObject(x.second->Type()); }}}; } [[nodiscard]] auto DirectoriesIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsTreeObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return x.second->IsTree(); }}}; } private: entries_t data_; }; FileRoot() noexcept = default; explicit FileRoot(bool ignore_special) noexcept : ignore_special_(ignore_special) {} // avoid type narrowing errors, but force explicit choice of underlying type explicit FileRoot(char const* /*root*/) noexcept { Logger::Log(LogLevel::Error, "FileRoot object instantiation must be explicit!"); } explicit FileRoot(char const* /*root*/, bool /*ignore_special*/) noexcept { Logger::Log(LogLevel::Error, "FileRoot object instantiation must be explicit!"); } explicit FileRoot(std::string root) noexcept : root_{absent_root_t{std::move(root)}} {} explicit FileRoot(std::string root, bool ignore_special) noexcept : root_{absent_root_t{std::move(root)}}, ignore_special_{ignore_special} {} explicit FileRoot(std::filesystem::path root) noexcept : root_{fs_root_t{std::move(root)}} {} FileRoot(std::filesystem::path root, bool ignore_special) noexcept : root_{fs_root_t{std::move(root)}}, ignore_special_{ignore_special} {} FileRoot(gsl::not_null const& cas, gsl::not_null const& tree, gsl::not_null const& root_entry, gsl::not_null const& storage_config, bool ignore_special = false) noexcept : root_{RootGit{cas, tree, root_entry}}, storage_config_{storage_config}, ignore_special_{ignore_special} {} explicit FileRoot(PrecomputedRoot precomputed) : root_{std::move(precomputed)} {} [[nodiscard]] static auto FromGit( gsl::not_null const& storage_config, std::filesystem::path const& repo_path, std::string const& git_tree_id, bool ignore_special = false) noexcept -> std::optional { // Check validity of hash early auto raw_tree_id = FromHexString(git_tree_id); if (not raw_tree_id) { return std::nullopt; } auto cas = GitCAS::Open(repo_path); if (not cas) { return std::nullopt; } // Read source tree root permissively from repository. Validity of // entries will be checked only when actually needed. auto tree = GitTree::Read( cas, git_tree_id, ignore_special, /*skip_checks=*/true); if (not tree) { return std::nullopt; } try { return FileRoot{ cas, std::make_shared(std::move(*tree)), std::make_shared( std::move(cas), std::move(*raw_tree_id), ObjectType::Tree), storage_config, ignore_special}; } catch (...) { return std::nullopt; } } /// \brief Return a complete description of the content of this root, if /// content fixed. This includes absent roots and any git-tree-based /// ignore-special roots. [[nodiscard]] auto ContentDescription() const noexcept -> std::optional { try { if (std::holds_alternative(root_)) { nlohmann::json j; j.push_back(ignore_special_ ? kGitTreeIgnoreSpecialMarker : kGitTreeMarker); // we need the root tree id, irrespective of ignore_special flag j.push_back(std::get(root_).tree->FileRootHash()); return j; } if (std::holds_alternative(root_)) { nlohmann::json j; j.push_back(ignore_special_ ? kGitTreeIgnoreSpecialMarker : kGitTreeMarker); j.push_back(std::get(root_)); return j; } } catch (std::exception const& ex) { Logger::Log(LogLevel::Debug, "Retrieving the description of a content-fixed root " "failed unexpectedly with:\n{}", ex.what()); } return std::nullopt; } /// \brief Get the source tree hash for a non-absent Git root. [[nodiscard]] auto GetTreeHash() const noexcept -> std::optional { if (auto const* root_git = std::get_if(&root_)) { return root_git->tree->FileRootHash(); } if (auto const* absent_root = std::get_if(&root_)) { return *absent_root; } return std::nullopt; } /// \brief Get the witnessing repository path for a non-absent Git root. [[nodiscard]] auto GetGitCasRoot() const noexcept -> std::optional { if (auto const* git_root = std::get_if(&root_)) { return git_root->cas->GetPath(); } return std::nullopt; } /// \brief Indicates that subsequent calls to `Exists()`, `IsFile()`, /// `IsDirectory()`, and `BlobType()` on valid contents of the same /// directory will be served without any additional file system lookups. [[nodiscard]] auto HasFastDirectoryLookup() const noexcept -> bool { return std::holds_alternative(root_); } /// \brief Check existence of path in non-absent source root. /// Invalid entries are treated as absent. [[nodiscard]] auto Exists(std::filesystem::path const& path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (path == ".") { return true; } if (auto entry = std::get(root_).tree->LookupEntryByPath(path)) { return IsNonSpecialObject(entry->Type()) or (not ignore_special_ and IsSymlinkObject(entry->Type()) and entry->Blob().has_value()); } } else if (std::holds_alternative(root_)) { auto root_path = std::get(root_) / path; auto exists = FileSystemManager::Exists(root_path); auto type = FileSystemManager::Type(root_path, /*allow_upwards=*/true); return (ignore_special_ ? exists and type and IsNonSpecialObject(*type) : exists); } return false; // absent roots cannot be interrogated locally } /// \brief Check if path is a file in non-absent source root. [[nodiscard]] auto IsFile( std::filesystem::path const& file_path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { return IsFileObject(entry->Type()); } } else if (std::holds_alternative(root_)) { return FileSystemManager::IsFile(std::get(root_) / file_path); } return false; // absent roots cannot be interrogated locally } /// \brief Check if path is a symlink in non-absent source root. /// Invalid entries are treated as missing. [[nodiscard]] auto IsSymlink( std::filesystem::path const& file_path) const noexcept -> bool { if (ignore_special_) { // skip unnecessary check return false; } if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { return IsSymlinkObject(entry->Type()) and entry->Blob().has_value(); } } else if (std::holds_alternative(root_)) { return FileSystemManager::IsNonUpwardsSymlink( std::get(root_) / file_path); } return false; // absent roots cannot be interrogated locally } /// \brief Check if path is a blob in non-absent source root. /// Invalid entries are treated as missing. [[nodiscard]] auto IsBlob( std::filesystem::path const& file_path) const noexcept -> bool { return IsFile(file_path) or IsSymlink(file_path); } /// \brief Check if path is a directory in non-absent source root. /// Does NOT verify entries. [[nodiscard]] auto IsDirectory( std::filesystem::path const& dir_path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (dir_path == ".") { return true; } if (auto entry = std::get(root_).tree->LookupEntryByPath( dir_path)) { return entry->IsTree(); } } else if (std::holds_alternative(root_)) { return FileSystemManager::IsDirectory(std::get(root_) / dir_path); } return false; // absent roots cannot be interrogated locally } /// \brief Read content of file or symlink from non-absent source root. /// Returns value only for valid entries. [[nodiscard]] auto ReadContent(std::filesystem::path const& file_path) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { if (IsBlobObject(entry->Type())) { // return blob content, if valid return entry->Blob(); } } } else if (std::holds_alternative(root_)) { auto full_path = std::get(root_) / file_path; if (auto type = FileSystemManager::Type(full_path, /*allow_upwards=*/false)) { // return content of file or non-upward symlink return IsSymlinkObject(*type) ? FileSystemManager::ReadSymlink(full_path) : FileSystemManager::ReadFile(full_path); } } return std::nullopt; } /// \brief Read entries of directory from non-absent source root. /// Returns value only for valid entries. [[nodiscard]] auto ReadDirectory(std::filesystem::path const& dir_path) const noexcept -> std::optional { try { if (std::holds_alternative(root_)) { auto const& git_root = std::get(root_); auto const& root_tree = git_root.tree; if (dir_path == ".") { if (ignore_special_ or GitTreeUtils::IsGitTreeValid(*storage_config_, git_root.root_entry)) { // stored root tree contains only valid entries, so use // it directly return DirectoryEntries{&(*root_tree)}; } Logger::Log( LogLevel::Debug, "Failed to read root directory of Git source tree {}", root_tree->FileRootHash()); return std::nullopt; } if (auto entry = root_tree->LookupEntryByPath(dir_path)) { if (ignore_special_ or GitTreeUtils::IsGitTreeValid(*storage_config_, entry)) { // simply get the GitTree reference; this is now valid // and it is guaranteed to not reread unnecessarily if (auto const& entry_tree = entry->Tree(ignore_special_)) { return DirectoryEntries{&(*entry_tree)}; } } Logger::Log( LogLevel::Debug, "Failed to read subdir {} of Git source tree {}", dir_path.string(), root_tree->FileRootHash()); return std::nullopt; } } else if (std::holds_alternative(root_)) { DirectoryEntries::pairs_t map{}; auto const subdir_path = std::get(root_) / dir_path; if (FileSystemManager::ReadDirectory( subdir_path, [&map](const auto& name, auto type) { map.emplace(name.string(), type); return true; }, /*allow_upwards=*/false, ignore_special_, /*log_failure_at=*/LogLevel::Debug)) { return DirectoryEntries{std::move(map)}; } Logger::Log(LogLevel::Debug, "Failed to read file source directory {}", subdir_path.string()); return std::nullopt; } } catch (std::exception const& ex) { Logger::Log(LogLevel::Debug, "Reading source directory {} failed with:\n{}", dir_path.string(), ex.what()); return std::nullopt; } return DirectoryEntries{DirectoryEntries::pairs_t{}}; } /// \brief Get type of blob at given path in non-absent root. /// Returns value only for valid entries. [[nodiscard]] auto BlobType(std::filesystem::path const& file_path) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { if (IsBlobObject(entry->Type())) { return entry->Type(); } } return std::nullopt; } if (std::holds_alternative(root_)) { auto type = FileSystemManager::Type( std::get(root_) / file_path, /*allow_upwards=*/true); if (type and IsBlobObject(*type)) { return type; } } return std::nullopt; } /// \brief Read a blob from a non-absent Git root based on its ID. /// Content of symlinks is validated. [[nodiscard]] auto ReadBlob(std::string const& blob_id, bool is_symlink = false) const noexcept -> std::optional { if (std::holds_alternative(root_)) { return std::get(root_).cas->ReadObject( blob_id, /*is_hex_id=*/true, /*as_valid_symlink=*/is_symlink); } return std::nullopt; } /// \brief Read tree from a non-absent Git root based on its ID. /// This should include all valid entry types. [[nodiscard]] auto ReadTree(std::string const& tree_id) const noexcept -> std::optional { try { if (std::holds_alternative(root_)) { auto const& cas = std::get(root_).cas; return GitTreeUtils::ReadValidGitCASTree( *storage_config_, tree_id, cas); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Unexpected failure while reading source tree {}:\n{}", tree_id, ex.what()); } return std::nullopt; } /// \brief Create LOCAL or KNOWN artifact. Does not check existence or /// validity for LOCAL. `file_path` must reference a blob. [[nodiscard]] auto ToArtifactDescription( HashFunction::Type hash_type, std::filesystem::path const& file_path, std::string const& repository) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { // if symlink, read content ahead of time to check validity std::optional blob{}; if (IsSymlinkObject(entry->Type())) { blob = entry->Blob(); if (not blob) { return std::nullopt; } } if (IsBlobObject(entry->Type())) { if (not ProtocolTraits::IsNative(hash_type)) { // read blob content, if not read already if (not blob) { blob = entry->Blob(); if (not blob) { return std::nullopt; } } auto compatible_hash = GitHashesConverter::Instance().RegisterGitEntry( entry->Hash(), *std::move(blob), repository); auto digest = ArtifactDigestFactory::Create(hash_type, compatible_hash, *entry->Size(), /*is_tree=*/false); if (not digest) { return std::nullopt; } return ArtifactDescription::CreateKnown( *std::move(digest), entry->Type()); } auto digest = ArtifactDigestFactory::Create(hash_type, entry->Hash(), *entry->Size(), /*is_tree=*/false); if (not digest) { return std::nullopt; } return ArtifactDescription::CreateKnown( *std::move(digest), entry->Type(), repository); } } return std::nullopt; } if (std::holds_alternative(root_)) { return ArtifactDescription::CreateLocal(file_path, repository); } return std::nullopt; // absent roots are neither LOCAL nor KNOWN } [[nodiscard]] auto IsAbsent() const noexcept -> bool { return std::holds_alternative(root_); } [[nodiscard]] auto GetAbsentTreeId() const noexcept -> std::optional { if (std::holds_alternative(root_)) { try { return std::get(root_); } catch (...) { return std::nullopt; } } return std::nullopt; } [[nodiscard]] auto IsPrecomputed() const noexcept -> bool { return std::holds_alternative(root_); } [[nodiscard]] auto GetPrecomputedDescription() const noexcept -> std::optional { if (auto const* precomputed = std::get_if(&root_)) { return *precomputed; } return std::nullopt; } [[nodiscard]] auto IgnoreSpecial() const noexcept -> bool { return ignore_special_; } /// \brief Parses a FileRoot from string. On errors, populates error_msg. /// \returns the FileRoot and optional local path (if the root is local), /// nullopt on errors. [[nodiscard]] static auto ParseRoot( gsl::not_null storage_config, std::string const& repo, std::string const& keyword, nlohmann::json const& root) -> expected>, std::string> { using ResultType = std::pair>; if ((not root.is_array()) or root.empty()) { return unexpected{fmt::format( "Expected {} for {} to be of the form [, ...], but " "found {}", keyword, repo, root.dump())}; } if (root[0] == "file") { if (root.size() != 2 or (not root[1].is_string())) { return unexpected{fmt::format( "\"file\" scheme expects precisely one string argument, " "but found {} for {} of repository {}", root.dump(), keyword, repo)}; } auto path = std::filesystem::path{root[1].get()}; return ResultType{FileRoot{path}, std::move(path)}; } if (root[0] == FileRoot::kGitTreeMarker) { bool const has_one_arg = root.size() == 2 and root[1].is_string(); bool const has_two_args = root.size() == 3 and root[1].is_string() and root[2].is_string(); if (not has_one_arg and not has_two_args) { return unexpected{fmt::format( "\"git tree\" scheme expects one or two string " "arguments, but found {} for {} of repository {}", root.dump(), keyword, repo)}; } if (root.size() == 3) { if (auto git_root = FileRoot::FromGit(storage_config, root[2], root[1])) { return ResultType{std::move(*git_root), std::nullopt}; } return unexpected{fmt::format( "Could not create file root for {}tree id {}", root.size() == 3 ? fmt::format("git repository {} and ", root[2]) : "", root[1])}; } // return absent root return ResultType{FileRoot{std::string{root[1]}}, std::nullopt}; } if (root[0] == FileRoot::kFileIgnoreSpecialMarker) { if (root.size() != 2 or (not root[1].is_string())) { return unexpected{fmt::format( "\"file ignore-special\" scheme expects precisely one " "string argument, but found {} for {} of repository {}", root.dump(), keyword, repo)}; } auto path = std::filesystem::path{root[1].get()}; return ResultType{FileRoot{path, /*ignore_special=*/true}, std::move(path)}; } if (root[0] == FileRoot::kGitTreeIgnoreSpecialMarker) { bool const has_one_arg = root.size() == 2 and root[1].is_string(); bool const has_two_args = root.size() == 3 and root[1].is_string() and root[2].is_string(); if (not has_one_arg and not has_two_args) { return unexpected{fmt::format( "\"git tree ignore-special\" scheme expects one or two " "string arguments, but found {} for {} of repository {}", root.dump(), keyword, repo)}; } if (root.size() == 3) { if (auto git_root = FileRoot::FromGit(storage_config, root[2], root[1], /*ignore_special=*/true)) { return ResultType{std::move(*git_root), std::nullopt}; } return unexpected{fmt::format( "Could not create ignore-special file root for {}tree id " "{}", root.size() == 3 ? fmt::format("git repository {} and ", root[2]) : "", root[1])}; } // return absent root return ResultType{ FileRoot{std::string{root[1]}, /*ignore_special=*/true}, std::nullopt}; } if (PrecomputedRoot::IsPrecomputedMarker(root[0])) { auto precomputed = PrecomputedRoot::Parse(root); if (not precomputed) { return unexpected{ fmt::format("While parsing {} for {} of repository{}:\n{}", root.dump(), keyword, repo, std::move(precomputed).error())}; } return ResultType{FileRoot{*std::move(precomputed)}, std::nullopt}; } return unexpected{fmt::format( "Unknown scheme in the specification {} of {} of repository {}", root.dump(), keyword, repo)}; } private: root_t root_; // For Git-based roots, keep a pointer to the StorageConfig const* storage_config_ = nullptr; // If set, forces lookups to ignore entries which are neither file nor // directories instead of erroring out. This means implicitly also that // there are no more fast tree lookups, i.e., tree traversal is a must. bool ignore_special_{}; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/file_storage.hpp000066400000000000000000000221511516554100600301030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_STORAGE_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_STORAGE_HPP #include #include #include #include #include #include #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" enum class StoreMode : std::uint8_t { // First thread to write conflicting file wins. FirstWins, // Last thread to write conflicting file wins, effectively overwriting // existing entries. NOTE: This might cause races if hard linking from // stored files due to an issue with the interaction of rename(2) and // link(2) (see: https://stackoverflow.com/q/69076026/1107763). LastWins }; struct FileStorageData final { /// Length of a subdirectory name. static constexpr size_t kDirectoryNameLength = 2; }; template > class FileStorage { public: explicit FileStorage(std::filesystem::path storage_root) noexcept : storage_root_{std::move(storage_root)} {} /// \brief Add file to storage. /// \returns true if file exists afterward. [[nodiscard]] auto AddFromFile(std::string const& id, std::filesystem::path const& source_path, bool is_owner = false) const noexcept -> bool { return AtomicAddFromFile(id, source_path, is_owner); } /// \brief Add bytes to storage. /// \returns true if file exists afterward. [[nodiscard]] auto AddFromBytes(std::string const& id, std::string const& bytes) const noexcept -> bool { return AtomicAddFromBytes(id, bytes); } /// \brief Determines the storage path of a blob identified by a hash value. /// The same sharding technique as used in git is applied, meaning, the hash /// value is separated into a directory part and file part. Two characters /// are used for the directory part, the rest for the file, which results in /// 256 possible directories. /// \param id The hash value of the blob. /// \returns The sharded file path. [[nodiscard]] auto GetPath(std::string const& id) const noexcept -> std::filesystem::path { return storage_root_ / id.substr(0, FileStorageData::kDirectoryNameLength) / id.substr(FileStorageData::kDirectoryNameLength, id.size() - FileStorageData::kDirectoryNameLength); } [[nodiscard]] auto StorageRoot() const noexcept -> std::filesystem::path const& { return storage_root_; } private: static constexpr bool kFdLess{kType == ObjectType::Executable}; std::filesystem::path storage_root_; /// \brief Add file to storage from file path via link or copy and rename. /// If a race-condition occurs, the winning thread will be the one /// performing the link/rename operation first or last, depending on kMode /// being set to FirstWins or LastWins, respectively. All threads will /// signal success. /// \returns true if file exists afterward. [[nodiscard]] auto AtomicAddFromFile(std::string const& id, std::filesystem::path const& path, bool is_owner) const noexcept -> bool { auto file_path = GetPath(id); if ((kMode == StoreMode::LastWins or not FileSystemManager::Exists(file_path)) and FileSystemManager::CreateDirectory(file_path.parent_path())) { auto direct_create = kMode == StoreMode::FirstWins and is_owner; auto create_directly = [&]() { // Entry does not exist and we are owner of the file (e.g., file // generated in the execution directory). Try to hard link it // directly or check its existence if it was created by now. return FileSystemManager::CreateFileHardlinkAs( path, file_path, /*log_failure_at=*/LogLevel::Debug) or FileSystemManager::IsFile(file_path); }; auto create_and_stage = [&]() { // Entry exists and we need to overwrite it, or we are not owner // of the file. Create the file in a process/thread-local // temporary path and stage it. auto unique_path = CreateUniquePath(file_path); return unique_path and CreateFileFromPath(*unique_path, path, is_owner) and StageFile(*unique_path, file_path); }; if (direct_create ? create_directly() : create_and_stage()) { Logger::Log( LogLevel::Trace, "created entry {}.", file_path.string()); return true; } } return FileSystemManager::IsFile(file_path); } /// \brief Add file to storage from bytes via write and atomic rename. /// If a race-condition occurs, the winning thread will be the one /// performing the rename operation first or last, depending on kMode being /// set to FirstWins or LastWins, respectively. All threads will signal /// success. /// \returns true if file exists afterward. [[nodiscard]] auto AtomicAddFromBytes( std::string const& id, std::string const& bytes) const noexcept -> bool { auto file_path = GetPath(id); if (kMode == StoreMode::LastWins or not FileSystemManager::Exists(file_path)) { auto unique_path = CreateUniquePath(file_path); if (unique_path and FileSystemManager::CreateDirectory(file_path.parent_path()) and CreateFileFromBytes(*unique_path, bytes) and StageFile(*unique_path, file_path)) { Logger::Log( LogLevel::Trace, "created entry {}.", file_path.string()); return true; } } return FileSystemManager::IsFile(file_path); } /// \brief Create file from file path. [[nodiscard]] static auto CreateFileFromPath( std::filesystem::path const& file_path, std::filesystem::path const& other_path, bool is_owner) noexcept -> bool { // For files owned by us (e.g., generated files from the execution // directory), prefer faster creation of hard links instead of a copy. // Copy executables without opening any writeable file descriptors in // this process to avoid those from being inherited by child processes. return (is_owner and FileSystemManager::CreateFileHardlinkAs( other_path, file_path)) or FileSystemManager::CopyFileAs( other_path, file_path, kFdLess); } /// \brief Create file from bytes. [[nodiscard]] static auto CreateFileFromBytes( std::filesystem::path const& file_path, std::string const& bytes) noexcept -> bool { // Write executables without opening any writeable file descriptors in // this process to avoid those from being inherited by child processes. return FileSystemManager::WriteFileAs( bytes, file_path, kFdLess); } /// \brief Stage file from source path to target path. [[nodiscard]] static auto StageFile( std::filesystem::path const& src_path, std::filesystem::path const& dst_path) noexcept -> bool { switch (kMode) { case StoreMode::FirstWins: // try rename source or delete it if the target already exists return FileSystemManager::Rename( src_path, dst_path, /*no_clobber=*/true) or (FileSystemManager::IsFile(dst_path) and FileSystemManager::RemoveFile(src_path)); case StoreMode::LastWins: return FileSystemManager::Rename(src_path, dst_path); } } }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_STORAGE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/file_system_manager.hpp000066400000000000000000001470201516554100600314600ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP #ifdef __unix__ #include #include #include #include #include #include #else #error "Non-unix is not supported yet" #endif #include #include #include // for errno #include #include #include #include // for std::fopen #include // std::exit, std::getenv #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/system/system.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/incremental_reader.hpp" #include "src/utils/cpp/path.hpp" namespace detail { static consteval auto BitWidth(int max_val) -> int { constexpr int kBitsPerByte = 8; int i = sizeof(max_val) * kBitsPerByte; while ((i-- > 0) and (((max_val >> i) & 0x01) == 0x00)) { // NOLINT } return i + 1; } } // namespace detail /// \brief Implements primitive file system functionality. /// Catches all exceptions for use with exception-free callers. class FileSystemManager { public: using ReadDirEntryFunc = std::function; using UseDirEntryFunc = std::function; class DirectoryAnchor { friend class FileSystemManager; public: DirectoryAnchor(DirectoryAnchor const&) = delete; auto operator=(DirectoryAnchor const&) -> DirectoryAnchor& = delete; auto operator=(DirectoryAnchor&&) -> DirectoryAnchor& = delete; ~DirectoryAnchor() noexcept { if (not restore_path_.empty()) { try { std::filesystem::current_path(restore_path_); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); } } } [[nodiscard]] auto GetRestorePath() const noexcept -> std::filesystem::path const& { return restore_path_; } private: std::filesystem::path const restore_path_; DirectoryAnchor() : restore_path_{FileSystemManager::GetCurrentDirectory()} {} DirectoryAnchor(DirectoryAnchor&&) = default; }; [[nodiscard]] static auto GetCurrentDirectory() noexcept -> std::filesystem::path { try { return std::filesystem::current_path(); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return std::filesystem::path{}; } } [[nodiscard]] static auto ChangeDirectory( std::filesystem::path const& dir) noexcept -> DirectoryAnchor { DirectoryAnchor anchor{}; try { std::filesystem::current_path(dir); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "changing directory to {} from anchor {}:\n{}", dir.string(), anchor.GetRestorePath().string(), e.what()); } return anchor; } /// \brief Returns true if the directory was created or existed before. [[nodiscard]] static auto CreateDirectory( std::filesystem::path const& dir) noexcept -> bool { return CreateDirectoryImpl(dir) != CreationStatus::Failed; } /// \brief Returns true if the directory was created by this call. [[nodiscard]] static auto CreateDirectoryExclusive( std::filesystem::path const& dir) noexcept -> bool { return CreateDirectoryImpl(dir) == CreationStatus::Created; } /// \brief Returns true if the file was created or existed before. [[nodiscard]] static auto CreateFile( std::filesystem::path const& file) noexcept -> bool { return CreateFileImpl(file) != CreationStatus::Failed; } /// \brief Returns true if the file was created by this call. [[nodiscard]] static auto CreateFileExclusive( std::filesystem::path const& file) noexcept -> bool { return CreateFileImpl(file) == CreationStatus::Created; } /// \brief Determine user home directory [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { char const* root{nullptr}; #ifdef __unix__ root = std::getenv("HOME"); if (root == nullptr) { root = getpwuid(getuid())->pw_dir; } #endif if (root == nullptr) { Logger::Log(LogLevel::Error, "Cannot determine user home directory."); std::exit(EXIT_FAILURE); } return root; } /// \brief We are POSIX-compliant, therefore we only care about the string /// value the symlinks points to, whether it exists or not, not the target /// type. As such, we don't distinguish directory or file targets. However, /// for maximum compliance, we use the directory symlink creator. [[nodiscard]] static auto CreateSymlink( std::filesystem::path const& to, std::filesystem::path const& link, LogLevel log_failure_at = LogLevel::Error) noexcept -> bool { try { if (not CreateDirectory(link.parent_path())) { Logger::Log(log_failure_at, "can not create directory {}", link.parent_path().string()); return false; } if (not RemoveFile(link)) { Logger::Log( log_failure_at, "can not remove file {}", link.string()); return false; } #ifdef __unix__ std::filesystem::create_directory_symlink(to, link); return std::filesystem::is_symlink(link); #else // For non-unix systems one would have to differentiate between file and // directory symlinks[1], which would require filesystem access and could lead // to inconsistencies due to order of creation of existing symlink targets. // [1]https://en.cppreference.com/w/cpp/filesystem/create_symlink #error "Non-unix is not supported yet" #endif } catch (std::exception const& e) { Logger::Log(log_failure_at, "symlinking {} to {}\n{}", to.string(), link.string(), e.what()); return false; } } [[nodiscard]] static auto CreateNonUpwardsSymlink( std::filesystem::path const& to, std::filesystem::path const& link, LogLevel log_failure_at = LogLevel::Error) noexcept -> bool { if (PathIsNonUpwards(to)) { return CreateSymlink(to, link, log_failure_at); } Logger::Log(log_failure_at, "symlink failure: target {} is not non-upwards", to.string()); return false; } /// \brief Try to create a hard link; return unit on success and a /// std::error_condition describing the failure. [[nodiscard]] static auto CreateFileHardlink( std::filesystem::path const& file_path, std::filesystem::path const& link_path, LogLevel log_failure_at = LogLevel::Error) noexcept -> expected { std::error_code ec{}; std::filesystem::create_hard_link(file_path, link_path, ec); if (not ec) { if (std::filesystem::is_regular_file(link_path)) { return std::monostate{}; } return unexpected(std::error_code{}); } Logger::Log(log_failure_at, "failed hard linking {} to {}: {}, {}", nlohmann::json(file_path.string()).dump(), nlohmann::json(link_path.string()).dump(), ec.value(), ec.message()); return unexpected(ec); } template requires(IsFileObject(kType)) [[nodiscard]] static auto CreateFileHardlinkAs( std::filesystem::path const& file_path, std::filesystem::path const& link_path, LogLevel log_failure_at = LogLevel::Error) noexcept -> bool { // Set permissions first (permissions are a property of the file) so // that the created link has the correct permissions as soon as the link // creation is finished. return SetFilePermissions(file_path, IsExecutableObject(kType)) and (not kSetEpochTime or SetEpochTime(file_path)) and CreateFileHardlink(file_path, link_path, log_failure_at); } template [[nodiscard]] static auto CreateFileHardlinkAs( std::filesystem::path const& file_path, std::filesystem::path const& link_path, ObjectType output_type) noexcept -> bool { switch (output_type) { case ObjectType::File: return CreateFileHardlinkAs( file_path, link_path); case ObjectType::Executable: return CreateFileHardlinkAs(file_path, link_path); case ObjectType::Tree: case ObjectType::Symlink: return false; } } [[nodiscard]] static auto Rename(std::filesystem::path const& src, std::filesystem::path const& dst, bool no_clobber = false) noexcept -> bool { if (no_clobber) { #ifdef __unix__ return link(src.c_str(), dst.c_str()) == 0 and unlink(src.c_str()) == 0; #else #error "Non-unix is not supported yet" #endif } try { std::filesystem::rename(src, dst); return true; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } /// \brief Copy file /// If argument fd_less is given, the copy will be performed in a child /// process to prevent polluting the parent with open writable file /// descriptors (which might be inherited by other children that keep them /// open and can cause EBUSY errors). [[nodiscard]] static auto CopyFile( std::filesystem::path const& src, std::filesystem::path const& dst, bool fd_less = false, std::filesystem::copy_options opt = std::filesystem::copy_options::overwrite_existing) noexcept -> bool { if (fd_less) { auto const* src_cstr = src.c_str(); auto const* dst_cstr = dst.c_str(); pid_t pid = ::fork(); if (pid == -1) { Logger::Log( LogLevel::Error, "Failed to copy file: cannot fork a child process."); return false; } if (pid == 0) { // In the child process, use low-level copies to avoid mallocs, // which removes the risk of deadlocks on certain combinations // of C++ standard library and libc. System::ExitWithoutCleanup(LowLevel::CopyFile( src_cstr, dst_cstr, opt == std::filesystem::copy_options::skip_existing)); } int status{}; ::waitpid(pid, &status, 0); // NOLINTNEXTLINE(hicpp-signed-bitwise) int retval = WEXITSTATUS(status); if (retval != 0) { Logger::Log(LogLevel::Error, "Failed copying file {} to {} with: {}", src.string(), dst.string(), LowLevel::ErrorToString(retval)); return false; } return true; } return CopyFileImpl(src, dst, opt); } template requires(IsFileObject(kType)) [[nodiscard]] static auto CopyFileAs( std::filesystem::path const& src, std::filesystem::path const& dst, bool fd_less = false, std::filesystem::copy_options opt = std::filesystem::copy_options::overwrite_existing) noexcept -> bool { return CopyFile(src, dst, fd_less, opt) and SetFilePermissions(dst, IsExecutableObject(kType)) and (not kSetEpochTime or SetEpochTime(dst)); } template [[nodiscard]] static auto CopyFileAs( std::filesystem::path const& src, std::filesystem::path const& dst, ObjectType type, bool fd_less = false, std::filesystem::copy_options opt = std::filesystem::copy_options::overwrite_existing) noexcept -> bool { switch (type) { case ObjectType::File: return CopyFileAs(src, dst, fd_less, opt); case ObjectType::Executable: return CopyFileAs(src, dst, fd_less, opt); case ObjectType::Symlink: return CopySymlinkAs( src, dst, opt == std::filesystem::copy_options::overwrite_existing); case ObjectType::Tree: break; } return false; } /// \brief Copy directory recursively [[nodiscard]] static auto CopyDirectoryImpl( std::filesystem::path const& src, std::filesystem::path const& dst) noexcept -> bool { try { // also checks existence if (not IsDirectory(src)) { Logger::Log(LogLevel::Error, "source {} does not exist or is not a directory", src.string()); return false; } // if dst does not exist, it is created, so only check if path // exists but is something else if (Exists(dst) and not IsDirectory(dst)) { Logger::Log(LogLevel::Error, "destination {} exists but it is not a directory", dst.string()); return false; } static constexpr auto kOptions = std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::recursive; std::filesystem::copy(src, dst, kOptions); return true; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "copying directory from {} to {}:\n{}", src.string(), dst.string(), e.what()); return false; } } /// \brief Create a symlink with option to set epoch time. template [[nodiscard]] static auto CreateSymlinkAs( std::filesystem::path const& to, std::filesystem::path const& link) noexcept -> bool { return CreateSymlink(to, link) and (not kSetEpochTime or SetEpochTime(link)); } /// \brief Create symlink copy at location, with option to overwriting any /// existing. Uses the content of src directly as the new target, whether /// src is a regular file (CAS entry) or another symlink. template [[nodiscard]] static auto CopySymlinkAs( std::filesystem::path const& src, std::filesystem::path const& dst, bool overwrite_existing = true) noexcept -> bool { try { if (overwrite_existing and Exists(dst) and not std::filesystem::remove(dst)) { Logger::Log(LogLevel::Debug, "could not overwrite existing path {}", dst.string()); return false; } if (std::filesystem::is_symlink(src)) { if (auto content = ReadSymlink(src)) { return CreateSymlinkAs(*content, dst); } } else { if (auto content = ReadFile(src)) { return CreateSymlinkAs(*content, dst); } } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "copying symlink from {} to {}:\n{}", src.string(), dst.string(), e.what()); } return false; } [[nodiscard]] static auto RemoveFile( std::filesystem::path const& file) noexcept -> bool { try { auto status = std::filesystem::symlink_status(file); if (not std::filesystem::exists(status)) { return true; } if (not std::filesystem::is_regular_file(status) and not std::filesystem::is_symlink(status)) { return false; } return std::filesystem::remove(file); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "removing file from {}:\n{}", file.string(), e.what()); return false; } } /// \brief Remove directory recursively. [[nodiscard]] static auto RemoveDirectory( std::filesystem::path const& dir) noexcept -> bool { try { auto status = std::filesystem::symlink_status(dir); if (not std::filesystem::exists(status)) { return true; } if (not std::filesystem::is_directory(status)) { return false; } // If it doesn't throw, it succeeds: std::ignore = std::filesystem::remove_all(dir); return true; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "removing directory {}:\n{}", dir.string(), e.what()); return false; } } /// \brief Returns if symlink is non-upwards, i.e., its string content path /// never passes itself in the directory tree. /// \param non_strict if set, do not check non-upwardness. Use with care! [[nodiscard]] static auto IsNonUpwardsSymlink( std::filesystem::path const& link, bool non_strict = false) noexcept -> bool { try { if (not std::filesystem::is_symlink(link)) { return false; } return non_strict or PathIsNonUpwards(std::filesystem::read_symlink(link)); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } /// \brief Follow a symlink chain without existence check on resulting path [[nodiscard]] static auto ResolveSymlinks( gsl::not_null path) noexcept -> bool { try { while (std::filesystem::is_symlink(*path)) { auto dest = std::filesystem::read_symlink(*path); *path = dest.is_relative() ? (std::filesystem::absolute(*path).parent_path() / dest) : dest; } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } return true; } [[nodiscard]] static auto Exists(std::filesystem::path const& path) noexcept -> bool { try { auto const status = std::filesystem::symlink_status(path); return std::filesystem::exists(status); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking for existence of path{}:\n{}", path.string(), e.what()); return false; } return true; } [[nodiscard]] static auto IsFile(std::filesystem::path const& file) noexcept -> bool { try { auto const status = std::filesystem::symlink_status(file); if (not std::filesystem::is_regular_file(status)) { return false; } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking if path {} corresponds to a file:\n{}", file.string(), e.what()); return false; } return true; } [[nodiscard]] static auto IsDirectory( std::filesystem::path const& dir) noexcept -> bool { try { auto const status = std::filesystem::symlink_status(dir); return std::filesystem::is_directory(status); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking if path {} corresponds to a directory:\n{}", dir.string(), e.what()); return false; } return true; } /// \brief Checks whether a path corresponds to an executable or not. /// \param[in] path Path to check /// \returns true if path corresponds to an executable object, false /// otherwise [[nodiscard]] static auto IsExecutable( std::filesystem::path const& path) noexcept -> bool { try { auto const status = std::filesystem::symlink_status(path); return std::filesystem::is_regular_file(status) and HasExecPermissions(status); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking if path {} corresponds to an executable:\n{}", path.string(), e.what()); return false; } return true; } /// \brief Gets type of object in path according to file system /// \param allow_upwards Do not enforce non-upwardness in symlinks. [[nodiscard]] static auto Type(std::filesystem::path const& path, bool allow_upwards = false) noexcept -> std::optional { try { auto const status = std::filesystem::symlink_status(path); if (std::filesystem::is_regular_file(status)) { if (HasExecPermissions(status)) { return ObjectType::Executable; } return ObjectType::File; } if (std::filesystem::is_directory(status)) { return ObjectType::Tree; } if (std::filesystem::is_symlink(status) and (allow_upwards or IsNonUpwardsSymlink(path))) { return ObjectType::Symlink; } if (std::filesystem::exists(status)) { Logger::Log(LogLevel::Debug, "object type for {} is not supported yet.", path.string()); } else { Logger::Log(LogLevel::Trace, "non-existing object path {}.", path.string()); } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking type of path {} failed with:\n{}", path.string(), e.what()); } return std::nullopt; } [[nodiscard]] static auto ReadFile( std::filesystem::path const& file) noexcept -> std::optional { auto const type = Type(file); if (not type) { Logger::Log(LogLevel::Trace, "{} can not be read because it is not a file.", file.string()); return std::nullopt; } return ReadFile(file, *type); } [[nodiscard]] static auto ReadFile(std::filesystem::path const& file, ObjectType type) noexcept -> std::optional { if (not IsFileObject(type)) { Logger::Log(LogLevel::Trace, "{} can not be read because it is not a file.", file.string()); return std::nullopt; } auto const to_read = IncrementalReader::FromFile(kChunkSize, file); if (not to_read.has_value()) { Logger::Log(LogLevel::Trace, "FileSystemManager: failed to create reader for {}\n{}", file.string(), to_read.error()); return std::nullopt; } try { std::string content{}; content.reserve(to_read->GetContentSize()); for (auto chunk : *to_read) { if (not chunk.has_value()) { Logger::Log(LogLevel::Error, "reading file failed {}:\n{}", file.string(), chunk.error()); return std::nullopt; } content.append(*chunk); } return content; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "reading file {}:\n{}", file.string(), e.what()); return std::nullopt; } } /// \brief Read a filesystem directory tree. /// \param ignore_special If true, do not error out when encountering /// symlinks. /// \param allow_upwards If true, do not enforce non-upwardness of symlinks. [[nodiscard]] static auto ReadDirectory( std::filesystem::path const& dir, ReadDirEntryFunc const& read_entry, bool allow_upwards = false, bool ignore_special = false, LogLevel log_failure_at = LogLevel::Debug) noexcept -> bool { try { for (auto const& entry : std::filesystem::directory_iterator{dir}) { ObjectType type{}; auto const status = entry.symlink_status(); if (std::filesystem::is_regular_file(status)) { if (HasExecPermissions(status)) { type = ObjectType::Executable; } else { type = ObjectType::File; } } else if (std::filesystem::is_directory(status)) { type = ObjectType::Tree; } // if not file, executable, or tree, ignore every other entry // type if asked to do so else if (ignore_special) { continue; } // if not already ignored, check symlinks and only add the // non-upwards ones else if (std::filesystem::is_symlink(status)) { if (not allow_upwards) { if (IsNonUpwardsSymlink(entry)) { type = ObjectType::Symlink; } else { Logger::Log( log_failure_at, "unsupported upwards symlink dir entry {}", entry.path().string()); return false; } } else { type = ObjectType::Symlink; } } else { Logger::Log(log_failure_at, "unsupported type for dir entry {}", entry.path().string()); return false; } if (not read_entry(entry.path().filename(), type)) { return false; } } } catch (std::exception const& ex) { Logger::Log( log_failure_at, "reading directory {} failed", dir.string()); return false; } return true; } /// \brief Read all entries recursively in a filesystem directory tree. /// \param dir root directory to traverse /// \param use_entry callback to call with found valid entries /// \param ignored_subdirs directory names to be ignored wherever found in /// the directory tree of dir. [[nodiscard]] static auto ReadDirectoryEntriesRecursive( std::filesystem::path const& dir, UseDirEntryFunc const& use_entry, std::unordered_set const& ignored_subdirs = {}) noexcept -> bool { try { // constructor of this iterator points to end by default; for (auto it = std::filesystem::recursive_directory_iterator(dir); it != std::filesystem::recursive_directory_iterator(); ++it) { // check for ignored subdirs if (std::filesystem::is_directory(it->symlink_status()) and ignored_subdirs.contains(*--it->path().end())) { it.disable_recursion_pending(); continue; } // use the entry if (not use_entry( it->path().lexically_relative(dir), std::filesystem::is_directory(it->symlink_status()))) { return false; } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading directory {} recursively failed", dir.string()); return false; } return true; } /// \brief Read the content of a symlink. [[nodiscard]] static auto ReadSymlink(std::filesystem::path const& link) -> std::optional { try { if (std::filesystem::is_symlink(link)) { return std::filesystem::read_symlink(link).string(); } Logger::Log(LogLevel::Debug, "{} can not be read because it is not a symlink.", link.string()); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading symlink {} failed:\n{}", link.string(), ex.what()); } return std::nullopt; } /// \brief Read the content of given file or symlink. [[nodiscard]] static auto ReadContentAtPath( std::filesystem::path const& fpath, ObjectType type) -> std::optional { try { if (IsSymlinkObject(type)) { return ReadSymlink(fpath); } if (IsFileObject(type)) { return ReadFile(fpath, type); } Logger::Log( LogLevel::Debug, "{} can not be read because it is neither a file nor symlink.", fpath.string()); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading content at path {} failed:\n{}", fpath.string(), ex.what()); } return std::nullopt; } /// \brief Write file /// If argument fd_less is given, the write will be performed in a child /// process to prevent polluting the parent with open writable file /// descriptors (which might be inherited by other children that keep them /// open and can cause EBUSY errors). [[nodiscard]] static auto WriteFile(std::string const& content, std::filesystem::path const& file, bool fd_less = false) noexcept -> bool { if (not CreateDirectory(file.parent_path())) { Logger::Log(LogLevel::Error, "can not create directory {}", file.parent_path().string()); return false; } if (fd_less) { auto const* file_cstr = file.c_str(); auto const* content_cstr = content.c_str(); auto content_size = content.size(); pid_t pid = ::fork(); if (pid == -1) { Logger::Log( LogLevel::Error, "Failed to write file: cannot fork a child process."); return false; } if (pid == 0) { // In the child process, use low-level writes to avoid mallocs, // which removes the risk of deadlocks on certain combinations // of C++ standard library and libc. System::ExitWithoutCleanup( LowLevel::WriteFile(content_cstr, content_size, file_cstr)); } int status{}; ::waitpid(pid, &status, 0); // NOLINTNEXTLINE(hicpp-signed-bitwise) int retval = WEXITSTATUS(status); if (retval != 0) { Logger::Log(LogLevel::Error, "Failed writing file {} with: {}", file.string(), LowLevel::ErrorToString(retval)); return false; } return true; } return WriteFileImpl(content, file); } template requires(IsFileObject(kType)) [[nodiscard]] static auto WriteFileAs(std::string const& content, std::filesystem::path const& file, bool fd_less = false) noexcept -> bool { return WriteFile(content, file, fd_less) and SetFilePermissions(file, IsExecutableObject(kType)) and (not kSetEpochTime or SetEpochTime(file)); } template [[nodiscard]] static auto WriteFileAs(std::string const& content, std::filesystem::path const& file, ObjectType output_type, bool fd_less = false) noexcept -> bool { switch (output_type) { case ObjectType::File: return WriteFileAs(content, file, fd_less); case ObjectType::Executable: return WriteFileAs(content, file, fd_less); case ObjectType::Symlink: return CreateSymlinkAs(content, file); case ObjectType::Tree: return false; } } [[nodiscard]] static auto IsRelativePath( std::filesystem::path const& path) noexcept -> bool { try { return path.is_relative(); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } [[nodiscard]] static auto IsAbsolutePath( std::filesystem::path const& path) noexcept -> bool { try { return path.is_absolute(); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } private: enum class CreationStatus : std::uint8_t { Created, Exists, Failed }; static constexpr std::size_t kChunkSize{256}; /// \brief Race condition free directory creation. /// Solves the TOCTOU issue. [[nodiscard]] static auto CreateDirectoryImpl( std::filesystem::path const& dir) noexcept -> CreationStatus { try { if (std::filesystem::is_directory( std::filesystem::symlink_status(dir))) { return CreationStatus::Exists; } if (std::filesystem::create_directories(dir)) { return CreationStatus::Created; } // It could be that another thread has created the directory right // after the current thread checked if it existed. For that reason, // we try to create it and check if it exists if create_directories // was not successful. if (std::filesystem::is_directory( std::filesystem::symlink_status(dir))) { return CreationStatus::Exists; } return CreationStatus::Failed; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return CreationStatus::Failed; } } /// \brief Race condition free file creation. /// Solves the TOCTOU issue via C11's std::fopen. [[nodiscard]] static auto CreateFileImpl( std::filesystem::path const& file) noexcept -> CreationStatus { try { if (std::filesystem::is_regular_file( std::filesystem::symlink_status(file))) { return CreationStatus::Exists; } if (gsl::owner fp = std::fopen(file.c_str(), "wx")) { std::fclose(fp); return CreationStatus::Created; } // It could be that another thread has created the file right after // the current thread checked if it existed. For that reason, we try // to create it and check if it exists if fopen() with exclusive bit // was not successful. if (std::filesystem::is_regular_file( std::filesystem::symlink_status(file))) { return CreationStatus::Exists; } return CreationStatus::Failed; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return CreationStatus::Failed; } } [[nodiscard]] static auto CopyFileImpl( std::filesystem::path const& src, std::filesystem::path const& dst, std::filesystem::copy_options opt = std::filesystem::copy_options::overwrite_existing) noexcept -> bool { try { // src should be an actual file if (std::filesystem::is_symlink(src)) { return false; } // Check that src and dst point to different filesystem entities: if (std::filesystem::weakly_canonical(src) == std::filesystem::weakly_canonical(dst)) { return true; } if (not RemoveFile(dst)) { Logger::Log( LogLevel::Error, "cannot remove file {}", dst.string()); return false; } return std::filesystem::copy_file(src, dst, opt); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "copying file from {} to {}:\n{}", src.string(), dst.string(), e.what()); return false; } } [[nodiscard]] static auto WriteFileImpl( std::string const& content, std::filesystem::path const& file) noexcept -> bool { if (not FileSystemManager::RemoveFile(file)) { Logger::Log( LogLevel::Error, "can not remove file {}", file.string()); return false; } try { std::ofstream writer{file}; if (not writer.is_open()) { Logger::Log( LogLevel::Error, "can not open file {}", file.string()); return false; } writer << content; writer.close(); return true; } catch (std::exception const& e) { Logger::Log( LogLevel::Error, "writing to {}:\n{}", file.string(), e.what()); return false; } } /// \brief Set special permissions for files. /// By default, we set to 0444 for non-executables and set to 0555 for /// executables. When we install or install-cas, we add the owner write /// permission to allow for, e.g., overwriting if we re-install the same /// target after a recompilation template static auto SetFilePermissions(std::filesystem::path const& path, bool is_executable) noexcept -> bool { try { using std::filesystem::perms; perms p{(kSetWritable ? perms::owner_write : perms::none) | perms::owner_read | perms::group_read | perms::others_read}; if (is_executable) { p |= perms::owner_exec | perms::group_exec | perms::others_exec; } std::filesystem::permissions(path, p); return true; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } /// \brief Set the last time of modification for a file (or symlink -- /// POSIX-only). static auto SetEpochTime(std::filesystem::path const& file_path) noexcept -> bool { static auto const kPosixEpochTime = System::GetPosixEpoch(); try { if (std::filesystem::is_symlink(file_path)) { // Because std::filesystem::last_write_time follows // symlinks, one has instead to manually call utimensat with // the AT_SYMLINK_NOFOLLOW flag. On non-POSIX systems, we // return false by default for symlinks. #ifdef __unix__ std::array times{}; // default is POSIX epoch if (utimensat(AT_FDCWD, file_path.c_str(), times.data(), AT_SYMLINK_NOFOLLOW) != 0) { Logger::Log(LogLevel::Error, "Call to utimensat for symlink {} failed with " "error: {}", file_path.string(), strerror(errno)); return false; } return true; #else Logger::Log( LogLevel::Warning, "Setting the last modification time attribute for a " "symlink is unsupported!"); return false; #endif } std::filesystem::last_write_time(file_path, kPosixEpochTime); return true; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return false; } } static auto HasExecPermissions( std::filesystem::file_status const& status) noexcept -> bool { try { namespace fs = std::filesystem; static constexpr auto kExecFlags = fs::perms::owner_exec bitor fs::perms::group_exec bitor fs::perms::others_exec; auto exec_perms = status.permissions() bitand kExecFlags; return exec_perms != fs::perms::none; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "checking for executable permissions failed with:\n{}", e.what()); } return false; } /// \brief Low-level copy and write operations. /// Those do not perform malloc operations, which removes the risk of /// deadlocks on certain combinations of C++ standard library and libc. /// Non-zero return values indicate errors, which can be decoded using /// \ref ErrorToString. class LowLevel { static constexpr std::size_t kDefaultChunkSize = 1024UL * 32; static constexpr int kWriteFlags = O_WRONLY | O_CREAT | O_TRUNC; // NOLINT static constexpr int kWritePerms = // 644 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; // NOLINT public: template [[nodiscard]] static auto CopyFile(char const* src, char const* dst, bool skip_existing) noexcept -> int { if (not skip_existing) { // remove dst if it exists if (unlink(dst) != 0 and errno != ENOENT) { return PackError(ERROR_OPEN_OUTPUT, errno); } } // NOLINTNEXTLINE(hicpp-signed-bitwise) auto write_flags = kWriteFlags | (skip_existing ? O_EXCL : 0); auto out = FdOpener{dst, write_flags, kWritePerms}; if (out.fd == -1) { if (skip_existing and errno == EEXIST) { return 0; } return PackError(ERROR_OPEN_OUTPUT, errno); } auto in = FdOpener{src, O_RDONLY}; if (in.fd == -1) { return PackError(ERROR_OPEN_INPUT, errno); } ssize_t len{}; std::array buf{}; while ((len = read(in.fd, buf.data(), buf.size())) > 0) { ssize_t wlen{}; ssize_t written_len{}; while (written_len < len and (wlen = write( out.fd, buf.data() + written_len, // NOLINT static_cast(len - written_len))) > 0) { written_len += wlen; } if (wlen < 0) { return PackError(ERROR_WRITE_OUTPUT, errno); } } if (len < 0) { return PackError(ERROR_READ_INPUT, errno); } return 0; } template [[nodiscard]] static auto WriteFile(char const* content, std::size_t size, char const* file) noexcept -> int { auto out = FdOpener{file, kWriteFlags, kWritePerms}; if (out.fd == -1) { return PackError(ERROR_OPEN_OUTPUT, errno); } std::size_t pos = 0; while (pos < size) { auto const write_len = std::min(kChunkSize, size - pos); auto const len = write(out.fd, content + pos, write_len); // NOLINT if (len < 0) { return PackError(ERROR_WRITE_OUTPUT, errno); } pos += static_cast(len); } return 0; } static auto ErrorToString(int retval) -> std::string { if (retval == 0) { return "no error"; } if ((retval & kSignalBit) == kSignalBit) { // NOLINT return fmt::format( "exceptional termination with return code {}", retval); } static auto strcode = [](int code) -> std::string { switch (code) { case ERROR_OPEN_INPUT: return "open() input file"; case ERROR_OPEN_OUTPUT: return "open() output file"; case ERROR_READ_INPUT: return "read() input file"; case ERROR_WRITE_OUTPUT: return "write() output file"; default: return "unknown operation"; } }; auto const [code, err] = UnpackError(retval); return fmt::format("{} failed with:\n{}: {} (probably)", strcode(code), err, strerror(err)); } private: enum ErrorCodes : std::uint8_t { ERROR_READ_INPUT, // read() input file failed ERROR_OPEN_INPUT, // open() input file failed ERROR_OPEN_OUTPUT, // open() output file failed ERROR_WRITE_OUTPUT, // write() output file failed LAST_ERROR_CODE // marker for first unused error code }; static constexpr int kSignalBit = 0x80; static constexpr int kAvailableBits = 7; // 8 bits - 1 signal bit static constexpr int kCodeWidth = detail::BitWidth(LAST_ERROR_CODE - 1); static constexpr int kCodeMask = (1 << kCodeWidth) - 1; // NOLINT static constexpr int kErrnoWidth = kAvailableBits - kCodeWidth; static constexpr int kErrnoMask = (1 << kErrnoWidth) - 1; // NOLINT // Open file descriptor and close on destruction. struct FdOpener { int fd; FdOpener(char const* path, int flags, int perms = 0) : fd{open(path, flags, perms)} {} // NOLINT FdOpener(FdOpener const&) = delete; FdOpener(FdOpener&&) = delete; auto operator=(FdOpener const&) = delete; auto operator=(FdOpener&&) = delete; ~FdOpener() { if (fd != -1) { close(fd); } } }; // encode to 8 bits with format static auto PackError(int code, int err) -> int { err &= kErrnoMask; // NOLINT if (code == 0 and err == 0) { err = kErrnoMask; } return (code << kErrnoWidth) | err; // NOLINT } static auto UnpackError(int retval) -> std::pair { int code = (retval >> kErrnoWidth) & kCodeMask; // NOLINT int err = retval & kErrnoMask; // NOLINT if (err == kErrnoMask) { err = 0; } return {code, err}; } }; // class LowLevel }; // class FileSystemManager #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_cas.cpp000066400000000000000000000171101516554100600270430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_cas.hpp" #include #include #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path.hpp" #ifndef BOOTSTRAP_BUILD_TOOL extern "C" { #include } namespace { [[nodiscard]] auto GitTypeToObjectType(git_object_t const& type) noexcept -> std::optional { switch (type) { case GIT_OBJECT_BLOB: return ObjectType::File; case GIT_OBJECT_TREE: return ObjectType::Tree; default: Logger::Log(LogLevel::Debug, "Unsupported git object type {}", git_object_type2string(type)); return std::nullopt; } } } // namespace #endif // BOOTSTRAP_BUILD_TOOL GitCAS::GitCAS() noexcept { GitContext::Create(); } auto GitCAS::Open(std::filesystem::path const& repo_path, LogLevel log_failure) noexcept -> GitCASPtr { #ifdef BOOTSTRAP_BUILD_TOOL return nullptr; #else try { auto result = std::make_shared(); // lock as git_repository API has no thread-safety guarantees static std::mutex repo_mutex{}; std::unique_lock lock{repo_mutex}; git_repository* repo_ptr = nullptr; if (git_repository_open_ext(&repo_ptr, repo_path.c_str(), GIT_REPOSITORY_OPEN_NO_SEARCH, nullptr) != 0 or repo_ptr == nullptr) { Logger::Log(log_failure, "Opening git repository {} failed with:\n{}", repo_path.string(), GitLastError()); return nullptr; } result->repo_.reset(repo_ptr); git_odb* odb_ptr = nullptr; if (git_repository_odb(&odb_ptr, result->repo_.get()) != 0 or odb_ptr == nullptr) { Logger::Log(log_failure, "Obtaining git object database {} failed with:\n{}", repo_path.string(), GitLastError()); return nullptr; } result->odb_.reset(odb_ptr); auto const git_path = git_repository_is_bare(result->repo_.get()) != 0 ? ToNormalPath(git_repository_path(result->repo_.get())) : ToNormalPath(git_repository_workdir(result->repo_.get())); result->git_path_ = std::filesystem::absolute(git_path); return std::const_pointer_cast(result); } catch (std::exception const& e) { Logger::Log(log_failure, "Unexpected failure opening git object database {}:\n{}", repo_path.string(), e.what()); return nullptr; } #endif } auto GitCAS::CreateEmpty() noexcept -> GitCASPtr { #ifdef BOOTSTRAP_BUILD_TOOL return nullptr; #else try { auto result = std::make_shared(); git_odb* odb_ptr{nullptr}; if (git_odb_new(&odb_ptr) != 0 or odb_ptr == nullptr) { Logger::Log(LogLevel::Error, "Creating an empty database failed with:\n{}", GitLastError()); return nullptr; } result->odb_.reset(odb_ptr); // retain odb pointer git_repository* repo_ptr = nullptr; if (git_repository_wrap_odb(&repo_ptr, result->odb_.get()) != 0 or repo_ptr == nullptr) { Logger::Log(LogLevel::Error, "Creating an empty repository failed with:\n{}", GitLastError()); return nullptr; } result->repo_.reset(repo_ptr); // retain repo pointer return std::const_pointer_cast(result); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Unexpected failure creating empty repository:\n{}", e.what()); return nullptr; } #endif } auto GitCAS::ReadObject(std::string const& id, bool is_hex_id, bool as_valid_symlink, LogLevel log_failure) const noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { if (not odb_) { return std::nullopt; } auto oid = GitObjectID(id, is_hex_id); if (not oid) { return std::nullopt; } git_odb_object* obj = nullptr; if (git_odb_read(&obj, odb_.get(), &oid.value()) != 0) { Logger::Log(log_failure, "Reading git object {} from database failed with:\n{}", is_hex_id ? id : ToHexString(id), GitLastError()); return std::nullopt; } std::string data(static_cast(git_odb_object_data(obj)), git_odb_object_size(obj)); auto obj_type = GitTypeToObjectType(git_odb_object_type(obj)); git_odb_object_free(obj); if (as_valid_symlink) { if (not obj_type) { return std::nullopt; } if (not IsTreeObject(*obj_type) and not PathIsNonUpwards(data)) { Logger::Log(log_failure, "Invalid git object {}: upwards symlink", is_hex_id ? id : ToHexString(id)); return std::nullopt; } } return data; } catch (std::exception const& e) { Logger::Log(log_failure, "Unexpected failure reading git object {}:\n{}", is_hex_id ? id : ToHexString(id), e.what()); return std::nullopt; } #endif } auto GitCAS::ReadHeader(std::string const& id, bool is_hex_id) const noexcept -> std::optional> { #ifndef BOOTSTRAP_BUILD_TOOL try { if (not odb_) { return std::nullopt; } auto oid = GitObjectID(id, is_hex_id); if (not oid) { return std::nullopt; } std::size_t size{}; git_object_t type{}; if (git_odb_read_header(&size, &type, odb_.get(), &oid.value()) != 0) { Logger::Log(LogLevel::Debug, "Reading git object header {} from database failed " "with:\n{}", is_hex_id ? id : ToHexString(id), GitLastError()); return std::nullopt; } if (auto obj_type = GitTypeToObjectType(type)) { return std::make_pair(size, *obj_type); } } catch (std::exception const& e) { Logger::Log(LogLevel::Debug, "Unexpected failure reading git object header {}:\n{}", is_hex_id ? id : ToHexString(id), e.what()); } #endif return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_cas.hpp000066400000000000000000000067641516554100600270650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" class GitCAS; using GitCASPtr = std::shared_ptr; /// \brief Git CAS that maintains its Git context. class GitCAS { public: [[nodiscard]] static auto Open( std::filesystem::path const& repo_path, LogLevel log_failure = LogLevel::Warning) noexcept -> GitCASPtr; [[nodiscard]] static auto CreateEmpty() noexcept -> GitCASPtr; GitCAS() noexcept; ~GitCAS() noexcept = default; // prohibit moves and copies GitCAS(GitCAS const&) = delete; GitCAS(GitCAS&& other) = delete; auto operator=(GitCAS const&) = delete; auto operator=(GitCAS&& other) = delete; [[nodiscard]] auto GetODB() const noexcept -> gsl::not_null { return odb_.get(); } [[nodiscard]] auto GetRepository() const noexcept -> gsl::not_null { return repo_.get(); } [[nodiscard]] auto GetPath() const noexcept -> std::filesystem::path const& { return git_path_; } /// \brief Read object from CAS. /// \param id The object id. /// \param is_hex_id Specify whether `id` is hex string or raw. /// \param as_valid_symlink Specify whether to treat any read blob as a /// valid symlink and check for non-upwardness. /// \param log_failure Log level at which to log failures accessing the /// object. [[nodiscard]] auto ReadObject(std::string const& id, bool is_hex_id, bool as_valid_symlink = false, LogLevel log_failure = LogLevel::Warning) const noexcept -> std::optional; /// \brief Read object header from CAS. /// \param id The object id. /// \param is_hex_id Specify whether `id` is hex string or raw. // Use with care. Quote from git2/odb.h:138: // Note that most backends do not support reading only the header of an // object, so the whole object will be read and then the header will be // returned. [[nodiscard]] auto ReadHeader(std::string const& id, bool is_hex_id) const noexcept -> std::optional>; private: std::unique_ptr odb_{nullptr, odb_closer}; std::unique_ptr repo_{ nullptr, repository_closer}; // git folder path of repo std::filesystem::path git_path_; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_context.cpp000066400000000000000000000025161516554100600277650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_context.hpp" #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" extern "C" { #include } GitContext::GitContext() noexcept { #ifndef BOOTSTRAP_BUILD_TOOL // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) initialized_ = git_libgit2_init() >= 0; if (not initialized_) { Logger::Log(LogLevel::Error, "initializing libgit2 failed"); } #endif } GitContext::~GitContext() noexcept { #ifndef BOOTSTRAP_BUILD_TOOL if (initialized_) { git_libgit2_shutdown(); } #endif } void GitContext::Create() noexcept { static GitContext git_state{}; Expects(git_state.initialized_); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_context.hpp000066400000000000000000000025261516554100600277730ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CONTEXT_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CONTEXT_HPP /// \brief Maintainer of a libgit2 state. /// Classes, static methods, and global functions dealing with Git operations /// should create a GitContext before using Git operations. class GitContext { public: // prohibit moves and copies GitContext(GitContext const&) = delete; GitContext(GitContext&& other) = delete; auto operator=(GitContext const&) = delete; auto operator=(GitContext&& other) = delete; static void Create() noexcept; ~GitContext() noexcept; private: GitContext() noexcept; bool initialized_{false}; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CONTEXT_HPPjust-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_repo.cpp000066400000000000000000002473031516554100600272530ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_repo.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/file_locking.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/tmp_dir.hpp" #ifndef NDEBUG #include "src/utils/cpp/gsl.hpp" #endif extern "C" { #include #include } #ifndef BOOTSTRAP_BUILD_TOOL namespace { /// \brief libgit2 file modes corresponding to non-special entries. std::unordered_set const kNonSpecialGitFileModes{ GIT_FILEMODE_BLOB, GIT_FILEMODE_BLOB_EXECUTABLE, GIT_FILEMODE_TREE}; [[nodiscard]] auto ToHexString(git_oid const& oid) -> std::optional { std::string hex_id(GIT_OID_HEXSZ, '\0'); if (git_oid_fmt(hex_id.data(), &oid) != 0) { return std::nullopt; } return hex_id; } [[nodiscard]] auto ToRawString(git_oid const& oid) noexcept -> std::optional { if (auto hex_id = ToHexString(oid)) { return FromHexString(*hex_id); } return std::nullopt; } /// \brief Returns true if mode corresponds to a supported object type. [[nodiscard]] auto GitFileModeIsNonSpecial(git_filemode_t const& mode) noexcept -> bool { return kNonSpecialGitFileModes.contains(mode); } [[nodiscard]] auto GitFileModeToObjectType(git_filemode_t const& mode) noexcept -> std::optional { switch (mode) { case GIT_FILEMODE_BLOB: return ObjectType::File; case GIT_FILEMODE_BLOB_EXECUTABLE: return ObjectType::Executable; case GIT_FILEMODE_TREE: return ObjectType::Tree; case GIT_FILEMODE_LINK: return ObjectType::Symlink; // condition not tested here default: { std::ostringstream str; str << std::oct << static_cast(mode); Logger::Log( LogLevel::Debug, "Unsupported git filemode {}", str.str()); return std::nullopt; } } } [[nodiscard]] auto GitTypeToObjectType(git_object_t const& type) -> std::optional { switch (type) { case GIT_OBJECT_BLOB: return ObjectType::File; case GIT_OBJECT_TREE: return ObjectType::Tree; default: Logger::Log(LogLevel::Debug, "Unsupported git object type {}", git_object_type2string(type)); return std::nullopt; } } [[nodiscard]] auto ObjectTypeToPerm(ObjectType type) noexcept -> std::string { switch (type) { case ObjectType::File: return "100644"; case ObjectType::Executable: return "100755"; case ObjectType::Tree: return "40000"; case ObjectType::Symlink: return "120000"; } return ""; // make gcc happy } #ifndef NDEBUG /// \brief Debug-level check that given tree entries are consistent. If needed, /// also check that the entries are in the underlying object database of the /// provided CAS instance. [[nodiscard]] auto ValidateEntries(GitRepo::tree_entries_t const& entries, GitCASPtr const& cas = nullptr) -> bool { return std::all_of(entries.begin(), entries.end(), [cas](auto entry) { auto const& [id, nodes] = entry; // if CAS given, check that the entry is in the object database if (cas != nullptr and not cas->ReadHeader(id, /*is_hex_id=*/false)) { return false; } // for a given raw id, either all entries are trees or none of them return std::all_of( nodes.begin(), nodes.end(), [](auto entry) { return IsTreeObject(entry.type); }) or std::none_of(nodes.begin(), nodes.end(), [](auto entry) { return IsTreeObject(entry.type); }); }); } #endif [[nodiscard]] auto flat_tree_walker_ignore_special(const char* /*root*/, const git_tree_entry* entry, void* payload) -> int { auto* entries = reinterpret_cast(payload); // NOLINT std::string name = git_tree_entry_name(entry); auto const* oid = git_tree_entry_id(entry); if (auto raw_id = ToRawString(*oid)) { if (not GitFileModeIsNonSpecial(git_tree_entry_filemode(entry))) { return 0; // allow, but not store } if (auto type = GitFileModeToObjectType(git_tree_entry_filemode(entry))) { // no need to test for symlinks, as no symlink entry will reach this (*entries)[*raw_id].emplace_back(std::move(name), *type); return 1; // return >=0 on success, 1 == skip subtrees (flat) } } Logger::Log(LogLevel::Debug, "Failed ignore_special walk for git tree entry: {}", name); return -1; // fail } [[nodiscard]] auto flat_tree_walker(const char* /*root*/, const git_tree_entry* entry, void* payload) -> int { auto* entries = reinterpret_cast(payload); // NOLINT std::string name = git_tree_entry_name(entry); auto const* oid = git_tree_entry_id(entry); if (auto raw_id = ToRawString(*oid)) { if (auto type = GitFileModeToObjectType(git_tree_entry_filemode(entry))) { // symlinks need to be checked in caller for non-upwardness (*entries)[*raw_id].emplace_back(std::move(name), *type); return 1; // return >=0 on success, 1 == skip subtrees (flat) } } Logger::Log(LogLevel::Debug, "Failed walk for git tree entry: {}", name); return -1; // fail } struct InMemoryODBBackend { git_odb_backend parent; GitRepo::tree_entries_t const* entries{nullptr}; // object headers std::unordered_map trees{}; // solid tree objects }; [[nodiscard]] auto backend_read_header(size_t* len_p, git_object_t* type_p, git_odb_backend* _backend, const git_oid* oid) -> int { if (len_p != nullptr and type_p != nullptr and _backend != nullptr and oid != nullptr) { auto* b = reinterpret_cast(_backend); // NOLINT if (auto id = ToRawString(*oid)) { if (auto it = b->trees.find(*id); it != b->trees.end()) { *type_p = GIT_OBJECT_TREE; *len_p = it->second.size(); return GIT_OK; } if (b->entries != nullptr) { if (auto it = b->entries->find(*id); it != b->entries->end()) { if (not it->second.empty()) { // pretend object is in database, size is ignored. *type_p = IsTreeObject(it->second.front().type) ? GIT_OBJECT_TREE : GIT_OBJECT_BLOB; *len_p = 0; return GIT_OK; } } } return GIT_ENOTFOUND; } } return GIT_ERROR; } [[nodiscard]] auto backend_read(void** data_p, size_t* len_p, git_object_t* type_p, git_odb_backend* _backend, const git_oid* oid) -> int { if (data_p != nullptr and len_p != nullptr and type_p != nullptr and _backend != nullptr and oid != nullptr) { auto* b = reinterpret_cast(_backend); // NOLINT if (auto id = ToRawString(*oid)) { if (auto it = b->trees.find(*id); it != b->trees.end()) { *type_p = GIT_OBJECT_TREE; *len_p = it->second.size(); *data_p = git_odb_backend_data_alloc(_backend, *len_p); if (*data_p == nullptr) { return GIT_ERROR; } std::memcpy(*data_p, it->second.data(), *len_p); return GIT_OK; } return GIT_ENOTFOUND; } } return GIT_ERROR; } [[nodiscard]] auto backend_exists(git_odb_backend* _backend, const git_oid* oid) -> int { if (_backend != nullptr and oid != nullptr) { auto* b = reinterpret_cast(_backend); // NOLINT if (auto id = ToRawString(*oid)) { return (b->entries != nullptr and b->entries->contains(*id)) or b->trees.contains(*id) ? 1 : 0; } } return GIT_ERROR; } [[nodiscard]] auto backend_write(git_odb_backend* _backend, const git_oid* oid, const void* data, std::size_t len, git_object_t type) -> int { if (data != nullptr and _backend != nullptr and oid != nullptr) { auto* b = reinterpret_cast(_backend); // NOLINT if (auto id = ToRawString(*oid)) { if (auto t = GitTypeToObjectType(type)) { std::string s(static_cast(data), len); if (type == GIT_OBJECT_TREE) { b->trees.emplace(std::move(*id), std::move(s)); return GIT_OK; } } } } return GIT_ERROR; } void backend_free(git_odb_backend* /*_backend*/) {} [[nodiscard]] auto CreateInMemoryODBParent() -> git_odb_backend { git_odb_backend b{}; b.version = GIT_ODB_BACKEND_VERSION; b.read_header = &backend_read_header; b.read = &backend_read; b.exists = &backend_exists; b.write = &backend_write; b.free = &backend_free; return b; } // A backend that can be used to read and create tree objects in-memory. auto const kInMemoryODBParent = CreateInMemoryODBParent(); struct FetchIntoODBBackend { git_odb_backend parent; git_odb* target_odb; // the odb where the fetch objects will end up into }; [[nodiscard]] auto fetch_backend_writepack(git_odb_writepack** _writepack, git_odb_backend* _backend, [[maybe_unused]] git_odb* odb, git_indexer_progress_cb progress_cb, void* progress_payload) -> int { Ensures(_backend != nullptr); auto* b = reinterpret_cast(_backend); // NOLINT return git_odb_write_pack( _writepack, b->target_odb, progress_cb, progress_payload); } [[nodiscard]] auto fetch_backend_exists(git_odb_backend* _backend, const git_oid* oid) -> int { Ensures(_backend != nullptr); auto* b = reinterpret_cast(_backend); // NOLINT return git_odb_exists(b->target_odb, oid); } void fetch_backend_free(git_odb_backend* /*_backend*/) {} [[nodiscard]] auto CreateFetchIntoODBParent() -> git_odb_backend { git_odb_backend b{}; b.version = GIT_ODB_BACKEND_VERSION; // only populate the functions needed b.writepack = &fetch_backend_writepack; // needed for fetch b.exists = &fetch_backend_exists; b.free = &fetch_backend_free; return b; } // A backend that can be used to fetch from the remote of another repository. auto const kFetchIntoODBParent = CreateFetchIntoODBParent(); // callback to remote fetch without an SSL certificate check const auto kCertificatePassthrough = [](git_cert* /*cert*/, int /*valid*/, const char* /*host*/, void* /*payload*/) -> int { return 0; }; } // namespace #endif // BOOTSTRAP_BUILD_TOOL auto GitRepo::Open(GitCASPtr git_cas) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else auto repo = GitRepo(std::move(git_cas)); if (not repo.git_cas_) { return std::nullopt; } return repo; #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::Open(std::filesystem::path const& repo_path) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else auto repo = GitRepo(repo_path); if (not repo.git_cas_) { return std::nullopt; } return repo; #endif // BOOTSTRAP_BUILD_TOOL } GitRepo::GitRepo(GitCASPtr git_cas) noexcept : git_cas_{std::move(git_cas)}, is_repo_fake_{true} {} GitRepo::GitRepo(std::filesystem::path const& repo_path) noexcept : git_cas_{GitCAS::Open(repo_path)}, is_repo_fake_{false} {} GitRepo::GitRepo(GitRepo&& other) noexcept : git_cas_{std::move(other.git_cas_)}, is_repo_fake_{other.is_repo_fake_} { other.git_cas_ = nullptr; } auto GitRepo::operator=(GitRepo&& other) noexcept -> GitRepo& { git_cas_ = std::move(other.git_cas_); is_repo_fake_ = other.is_repo_fake_; other.git_cas_ = nullptr; return *this; } auto GitRepo::InitAndOpen(std::filesystem::path const& repo_path, bool is_bare) noexcept -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL try { static std::mutex repo_mutex{}; std::unique_lock lock{repo_mutex}; GitContext::Create(); // initialize libgit2 git_repository* tmp_repo{nullptr}; int err = 0; std::string err_mess{}; for (auto attempt = 0U; attempt < kGitLockNumTries; ++attempt) { // check if init is needed or has already happened in another // process if (git_repository_open_ext(nullptr, repo_path.c_str(), GIT_REPOSITORY_OPEN_NO_SEARCH, nullptr) == 0) { return GitRepo(repo_path); // success } // Initialization must be guarded across processes trying to // initialize the same repo. if (auto const lock_file = LockFile::Acquire( repo_path.parent_path() / "init_open.lock", /*is_shared=*/false)) { err = git_repository_init(&tmp_repo, repo_path.c_str(), static_cast(is_bare)); } else { Logger::Log(LogLevel::Error, "Initializing git repository {} failed to " "acquire lock."); return std::nullopt; } if (err == 0) { git_repository_free(tmp_repo); return GitRepo(repo_path); // success } err_mess = GitLastError(); // store last error message git_repository_free(tmp_repo); // cleanup before next attempt // only retry if failure is due to locking if (err != GIT_ELOCKED) { break; } // repo still not created, so sleep and try again std::this_thread::sleep_for( std::chrono::milliseconds(kGitLockWaitTime)); } Logger::Log(LogLevel::Error, "Initializing git repository {} failed with:\n{}", repo_path.string(), err_mess); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Initializing git repository {} failed with:\n{}", repo_path.string(), ex.what()); } #endif // BOOTSTRAP_BUILD_TOOL return std::nullopt; } auto GitRepo::GetGitCAS() const noexcept -> GitCASPtr { return git_cas_; } auto GitRepo::CommitDirectory(std::filesystem::path const& dir, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // only possible for real repository! if (IsRepoFake()) { std::invoke(*logger, "Cannot commit directory using a fake repository!", true /*fatal*/); return std::nullopt; } // Due to limitations of Git in general, and libgit2 in particular, by // which updating the index with entries that have Git-specific magic // names is cumbersome, if at all possible, we resort to creating // manually the tree to be committed from the given subdirectory by // recursively creating and writing to the object database all the blobs // and subtrees. // get tree containing the subdirectory entries auto raw_id = CreateTreeFromDirectory(dir, logger); if (not raw_id) { return std::nullopt; } // get tree oid git_oid tree_oid; if (git_oid_fromraw(&tree_oid, reinterpret_cast( // NOLINT raw_id->data())) != 0) { std::invoke(*logger, fmt::format("Subdir tree object id parsing in git " "repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } // set committer signature git_signature* signature_ptr{nullptr}; if (git_signature_new( &signature_ptr, "Nobody", "nobody@example.org", 0, 0) != 0) { std::invoke(*logger, fmt::format("Creating signature in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_signature_free(signature_ptr); return std::nullopt; } auto signature = std::unique_ptr( signature_ptr, signature_closer); // get tree object git_tree* tree_ptr = nullptr; if (git_tree_lookup(&tree_ptr, git_cas_->GetRepository(), &tree_oid) != 0) { std::invoke( *logger, fmt::format("Tree lookup in git repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_tree_free(tree_ptr); return std::nullopt; } auto tree = std::unique_ptr( tree_ptr, tree_closer); // commit the tree containing the staged files git_buf buffer = GIT_BUF_INIT_CONST(NULL, 0); git_message_prettify(&buffer, message.c_str(), 0, '#'); git_oid commit_oid; // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) if (git_commit_create_v(&commit_oid, git_cas_->GetRepository(), "HEAD", signature.get(), signature.get(), nullptr, buffer.ptr, tree.get(), 0) != 0) { std::invoke( *logger, fmt::format("Git commit in repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); git_buf_dispose(&buffer); return std::nullopt; } std::string commit_hash{git_oid_tostr_s(&commit_oid)}; git_buf_dispose(&buffer); return commit_hash; // success! } catch (std::exception const& ex) { std::invoke(*logger, fmt::format("Commit subdir failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::KeepTag(std::string const& commit, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // only possible for real repository! if (IsRepoFake()) { std::invoke(*logger, "Cannot tag commits using a fake repository!", true /*fatal*/); return std::nullopt; } // get commit spec git_object* target_ptr{nullptr}; if (git_revparse_single( &target_ptr, git_cas_->GetRepository(), commit.c_str()) != 0) { std::invoke(*logger, fmt::format("Rev-parse commit {} in repository {} " "failed with:\n{}", commit, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); git_object_free(target_ptr); return std::nullopt; } auto target = std::unique_ptr( target_ptr, object_closer); // set tagger signature git_signature* tagger_ptr{nullptr}; if (git_signature_new( &tagger_ptr, "Nobody", "nobody@example.org", 0, 0) != 0) { std::invoke(*logger, fmt::format("Creating signature in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_signature_free(tagger_ptr); return std::nullopt; } auto tagger = std::unique_ptr( tagger_ptr, signature_closer); // create tag git_oid oid; auto name = fmt::format("keep-{}", commit); git_strarray tag_names{}; // check if tag hasn't already been added by another process if (git_tag_list_match( &tag_names, name.c_str(), git_cas_->GetRepository()) == 0 and tag_names.count > 0) { git_strarray_dispose(&tag_names); return name; // success! } git_strarray_dispose(&tag_names); // free any allocated unused space std::size_t max_attempts = kGitLockNumTries; // number of tries int err = 0; std::string err_mess{}; while (max_attempts > 0) { --max_attempts; err = git_tag_create(&oid, git_cas_->GetRepository(), name.c_str(), target.get(), tagger.get(), message.c_str(), 1 /*force*/); if (err == 0) { return name; // success! } err_mess = GitLastError(); // store last error message // only retry if failure is due to locking if (err != GIT_ELOCKED) { break; } // check if tag hasn't already been added by another process if (git_tag_list_match(&tag_names, name.c_str(), git_cas_->GetRepository()) == 0 and tag_names.count > 0) { git_strarray_dispose(&tag_names); return name; // success! } git_strarray_dispose( &tag_names); // free any allocated unused space // tag still not in, so sleep and try again std::this_thread::sleep_for( std::chrono::milliseconds(kGitLockWaitTime)); } std::invoke( *logger, fmt::format("Tag creation in git repository {} failed with:\n{}", git_cas_->GetPath().string(), err_mess), true /*fatal*/); return std::nullopt; } catch (std::exception const& ex) { std::invoke(*logger, fmt::format("Keep tag failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetHeadCommit(anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // only possible for real repository! if (IsRepoFake()) { std::invoke(*logger, "Cannot access HEAD ref using a fake repository!", true /*fatal*/); return std::nullopt; } // get root commit id git_oid head_oid; if (git_reference_name_to_id( &head_oid, git_cas_->GetRepository(), "HEAD") != 0) { std::invoke(*logger, fmt::format("Retrieving head commit in git repository " "{} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } return std::string(git_oid_tostr_s(&head_oid)); } catch (std::exception const& ex) { std::invoke(*logger, fmt::format("Get head commit failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::FetchFromPath(std::shared_ptr cfg, std::string const& repo_path, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool { #ifdef BOOTSTRAP_BUILD_TOOL return false; #else try { // only possible for real repository! if (IsRepoFake()) { std::invoke(*logger, "Cannot fetch commit using a fake repository!", true /*fatal*/); return false; } // create remote from repo git_remote* remote_ptr{nullptr}; if (git_remote_create_anonymous(&remote_ptr, git_cas_->GetRepository(), repo_path.c_str()) != 0) { std::invoke(*logger, fmt::format("Creating remote {} for local repository " "{} failed with:\n{}", repo_path, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_remote_free(remote_ptr); return false; } // wrap remote object auto remote = std::unique_ptr( remote_ptr, remote_closer); // get a well-defined config file if (not cfg) { // get config snapshot of current repo git_config* cfg_ptr{nullptr}; if (git_repository_config_snapshot( &cfg_ptr, git_cas_->GetRepository()) != 0) { std::invoke(*logger, fmt::format("Retrieving config object in fetch " "from path failed with:\n{}", GitLastError()), true /*fatal*/); return false; } // share pointer with caller cfg.reset(cfg_ptr, config_closer); } // define default fetch options git_fetch_options fetch_opts{}; git_fetch_options_init(&fetch_opts, GIT_FETCH_OPTIONS_VERSION); // no proxy fetch_opts.proxy_opts.type = GIT_PROXY_NONE; // no SSL verification fetch_opts.callbacks.certificate_check = kCertificatePassthrough; // disable update of the FETCH_HEAD pointer fetch_opts.update_fetchhead = 0; // setup fetch refspecs array GitStrArray refspecs_array_obj; if (branch) { // make sure we check for tags as well refspecs_array_obj.AddEntry(fmt::format("+refs/tags/{}", *branch)); refspecs_array_obj.AddEntry(fmt::format("+refs/heads/{}", *branch)); } auto const refspecs_array = refspecs_array_obj.Get(); if (git_remote_fetch( remote.get(), &refspecs_array, &fetch_opts, nullptr) != 0) { std::invoke( *logger, fmt::format( "Fetching {} in local repository {} failed with:\n{}", branch ? fmt::format("branch {}", *branch) : "all", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return false; } return true; // success! } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Fetch {} from local repository {} failed with:\n{}", branch ? fmt::format("branch {}", *branch) : "all", repo_path, ex.what()), true /*fatal*/); return false; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::KeepTree(std::string const& tree_id, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // only possible for real repository! if (IsRepoFake()) { std::invoke(*logger, "Cannot commit and tag a tree using a fake repository!", true /*fatal*/); return std::nullopt; } // get tree oid git_oid tree_oid; if (git_oid_fromstr(&tree_oid, tree_id.c_str()) != 0) { std::invoke(*logger, fmt::format("Tree ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } // get tree object from oid git_object* target_ptr{nullptr}; if (git_object_lookup(&target_ptr, git_cas_->GetRepository(), &tree_oid, GIT_OBJECT_TREE) != 0) { std::invoke(*logger, fmt::format("Object lookup for tree {} in repository " "{} failed with:\n{}", tree_id, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); git_object_free(target_ptr); return std::nullopt; } auto target = std::unique_ptr( target_ptr, object_closer); // set signature git_signature* signature_ptr{nullptr}; if (git_signature_new( &signature_ptr, "Nobody", "nobody@example.org", 0, 0) != 0) { std::invoke(*logger, fmt::format("Creating signature in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_signature_free(signature_ptr); return std::nullopt; } auto signature = std::unique_ptr( signature_ptr, signature_closer); // create tag git_oid oid; auto name = fmt::format("keep-{}", tree_id); git_strarray tag_names{}; // check if tag hasn't already been added by another process if (git_tag_list_match( &tag_names, name.c_str(), git_cas_->GetRepository()) == 0 and tag_names.count > 0) { git_strarray_dispose(&tag_names); return name; // success! } git_strarray_dispose(&tag_names); // free any allocated unused space size_t max_attempts = kGitLockNumTries; // number of tries int err = 0; std::string err_mess{}; while (max_attempts > 0) { --max_attempts; err = git_tag_create(&oid, git_cas_->GetRepository(), name.c_str(), target.get(), /*tree*/ signature.get(), /*tagger*/ message.c_str(), 1 /*force*/); if (err == 0) { return name; // success! } err_mess = GitLastError(); // store last error message // only retry if failure is due to locking if (err != GIT_ELOCKED) { break; } // check if tag hasn't already been added by another process if (git_tag_list_match(&tag_names, name.c_str(), git_cas_->GetRepository()) == 0 and tag_names.count > 0) { git_strarray_dispose(&tag_names); return name; // success! } git_strarray_dispose( &tag_names); // free any allocated unused space // tag still not in, so sleep and try again std::this_thread::sleep_for( std::chrono::milliseconds(kGitLockWaitTime)); } std::invoke(*logger, fmt::format("Tag creation for tree {} in git repository {} " "failed with:\n{}", tree_id, git_cas_->GetPath().string(), err_mess), true /*fatal*/); return std::nullopt; } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Keep tree {} failed with:\n{}", tree_id, ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetSubtreeFromCommit(std::string const& commit, std::string const& subdir, anon_logger_ptr const& logger) noexcept -> expected { #ifdef BOOTSTRAP_BUILD_TOOL return unexpected{GitLookupError::Fatal}; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log( LogLevel::Debug, "Subtree id retrieval from commit called on a real repository"); } // get commit object git_oid commit_oid; if (git_oid_fromstr(&commit_oid, commit.c_str()) != 0) { std::invoke(*logger, fmt::format("Commit ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return unexpected{GitLookupError::Fatal}; } git_commit* commit_ptr{nullptr}; if (git_commit_lookup( &commit_ptr, git_cas_->GetRepository(), &commit_oid) != 0) { std::invoke(*logger, fmt::format("Retrieving commit {} in git repository {} " "failed with:\n{}", commit, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_commit_free(commit_ptr); return unexpected{GitLookupError::NotFound}; // non-fatal failure } auto commit_obj = std::unique_ptr( commit_ptr, commit_closer); // get tree of commit git_tree* tree_ptr{nullptr}; if (git_commit_tree(&tree_ptr, commit_obj.get()) != 0) { std::invoke(*logger, fmt::format("Retrieving tree for commit {} in git " "repository {} failed with:\n{}", commit, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_tree_free(tree_ptr); return unexpected{GitLookupError::Fatal}; } auto tree = std::unique_ptr( tree_ptr, tree_closer); if (subdir != ".") { // get hash for actual subdir git_tree_entry* subtree_entry_ptr{nullptr}; if (git_tree_entry_bypath( &subtree_entry_ptr, tree.get(), subdir.c_str()) != 0) { std::invoke(*logger, fmt::format("Retrieving subtree at {} in git " "repository {} failed with:\n{}", subdir, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_tree_entry_free(subtree_entry_ptr); return unexpected{GitLookupError::Fatal}; } auto subtree_entry = std::unique_ptr( subtree_entry_ptr, tree_entry_closer); std::string subtree_hash{ git_oid_tostr_s(git_tree_entry_id(subtree_entry.get()))}; return subtree_hash; // success } // if no subdir, get hash from tree std::string tree_hash{git_oid_tostr_s(git_tree_id(tree.get()))}; return tree_hash; // success } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Get subtree from commit failed with:\n{}", ex.what()), true /*fatal*/); return unexpected{GitLookupError::Fatal}; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetSubtreeFromTree(std::string const& tree_id, std::string const& subdir, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // check if subdir is not trivial if (subdir != ".") { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Subtree id retrieval from tree called on a real " "repository"); } // get tree object from tree id git_oid tree_oid; if (git_oid_fromstr(&tree_oid, tree_id.c_str()) != 0) { std::invoke(*logger, fmt::format("Tree ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } git_tree* tree_ptr{nullptr}; if (git_tree_lookup( &tree_ptr, git_cas_->GetRepository(), &tree_oid) != 0) { std::invoke(*logger, fmt::format("Retrieving tree {} in git repository " "{} failed with:\n{}", tree_id, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_tree_free(tree_ptr); return std::nullopt; } auto tree = std::unique_ptr( tree_ptr, tree_closer); // get hash for actual subdir git_tree_entry* subtree_entry_ptr{nullptr}; if (git_tree_entry_bypath( &subtree_entry_ptr, tree.get(), subdir.c_str()) != 0) { std::invoke(*logger, fmt::format("Retrieving subtree at {} in git " "repository {} failed with:\n{}", subdir, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_tree_entry_free(subtree_entry_ptr); return std::nullopt; } auto subtree_entry = std::unique_ptr( subtree_entry_ptr, tree_entry_closer); std::string subtree_hash{ git_oid_tostr_s(git_tree_entry_id(subtree_entry.get()))}; return subtree_hash; } // if no subdir, return given tree hash return tree_id; } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Get subtree from tree failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetSubtreeFromPath(std::filesystem::path const& fpath, std::string const& head_commit, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log( LogLevel::Debug, "Subtree id retrieval from path called on a real repository"); } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { std::invoke( *logger, fmt::format("While getting repo root from path:\n{}", msg), fatal); }); // find root dir of this repository auto root = GetRepoRootFromPath(fpath, wrapped_logger); if (root == std::nullopt) { return std::nullopt; } // setup wrapped logger wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { std::invoke(*logger, fmt::format("While going subtree hash retrieval " "from path:\n{}", msg), fatal); }); // find relative path from root to given path auto subdir = std::filesystem::relative(fpath, *root).string(); // get subtree from head commit and subdir auto res = GetSubtreeFromCommit(head_commit, subdir, wrapped_logger); if (not res) { return std::nullopt; } return *std::move(res); } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Get subtree from path failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::CheckCommitExists(std::string const& commit, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Commit lookup called on a real repository"); } // lookup commit in current repo state git_oid commit_oid; if (git_oid_fromstr(&commit_oid, commit.c_str()) != 0) { std::invoke(*logger, fmt::format("Commit ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } git_commit* commit_obj = nullptr; int const lookup_res = git_commit_lookup( &commit_obj, git_cas_->GetRepository(), &commit_oid); if (lookup_res != 0) { if (lookup_res == GIT_ENOTFOUND) { // cleanup resources git_commit_free(commit_obj); return false; // commit not found } // failure std::invoke(*logger, fmt::format("Lookup of commit {} in git repository {} " "failed with:\n{}", commit, git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_commit_free(commit_obj); return std::nullopt; } // cleanup resources git_commit_free(commit_obj); return true; // commit exists } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Check commit exists failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetRepoRootFromPath(std::filesystem::path const& fpath, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { GitContext::Create(); // initialize libgit2 git_buf buffer = GIT_BUF_INIT_CONST(NULL, 0); auto res = git_repository_discover(&buffer, fpath.c_str(), 0, nullptr); if (res != 0) { if (res == GIT_ENOTFOUND) { git_buf_dispose(&buffer); return std::filesystem::path{}; // empty path cause nothing // found } // failure std::invoke(*logger, fmt::format("Repository root search failed at path {} " "with:\n{}!", fpath.string(), GitLastError()), true /*fatal*/); git_buf_dispose(&buffer); return std::nullopt; } // found root repo path std::string result{buffer.ptr}; git_buf_dispose(&buffer); // normalize root result auto actual_root = std::filesystem::path{result}.parent_path(); // remove trailing "/" if (actual_root.parent_path() / ".git" == actual_root) { return actual_root.parent_path(); // remove ".git" folder from path } return actual_root; } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Get repo root from path failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::CheckTreeExists(std::string const& tree_id, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Tree lookup called on a real repository"); } // get git oid git_oid tree_oid; if (git_oid_fromstr(&tree_oid, tree_id.c_str()) != 0) { std::invoke(*logger, fmt::format("Tree ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } // get tree object git_tree* tree_ptr = nullptr; int const lookup_res = git_tree_lookup(&tree_ptr, git_cas_->GetRepository(), &tree_oid); git_tree_free(tree_ptr); if (lookup_res != 0) { if (lookup_res == GIT_ENOTFOUND) { return false; // tree not found } std::invoke( *logger, fmt::format("Tree lookup in git repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } return true; // tree found } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Check tree exists failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::CheckBlobExists(std::string const& blob_id, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Blob lookup called on a real repository"); } // get git oid git_oid blob_oid; if (git_oid_fromstr(&blob_oid, blob_id.c_str()) != 0) { std::invoke(*logger, fmt::format("Blob ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } // get blob object git_blob* blob_ptr = nullptr; int const lookup_res = git_blob_lookup(&blob_ptr, git_cas_->GetRepository(), &blob_oid); git_blob_free(blob_ptr); if (lookup_res != 0) { if (lookup_res == GIT_ENOTFOUND) { return false; // blob not found } std::invoke( *logger, fmt::format("Blob lookup in git repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } return true; // blob found } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Check blob exists failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::TryReadBlob(std::string const& blob_id, anon_logger_ptr const& logger) noexcept -> std::pair> { #ifdef BOOTSTRAP_BUILD_TOOL return std::pair(false, std::nullopt); #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Blob lookup called on a real repository"); } // get git oid git_oid blob_oid; if (git_oid_fromstr(&blob_oid, blob_id.c_str()) != 0) { std::invoke(*logger, fmt::format("Blob ID parsing in git repository {} " "failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::pair(false, std::nullopt); } // get blob object git_blob* blob_ptr = nullptr; int const lookup_res = git_blob_lookup(&blob_ptr, git_cas_->GetRepository(), &blob_oid); git_blob_free(blob_ptr); if (lookup_res != 0) { if (lookup_res == GIT_ENOTFOUND) { return std::pair(true, std::nullopt); // blob not found } std::invoke( *logger, fmt::format("Blob lookup in git repository {} failed with:\n{}", git_cas_->GetPath().string(), GitLastError()), true /*fatal*/); return std::pair(false, std::nullopt); } // get data of found blob if (auto data = git_cas_->ReadObject(blob_id, /*is_hex_id=*/true)) { return std::pair(true, std::move(*data)); } std::invoke(*logger, fmt::format("Failed to read target for blob {} in git " "repository {}", blob_id, git_cas_->GetPath().string()), true /*fatal*/); return std::pair(false, std::nullopt); } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Check blob exists failed with:\n{}", ex.what()), true /*fatal*/); return std::pair(false, std::nullopt); } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::WriteBlob(std::string const& content, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Blob writer called on a real repository"); } git_oid blob_oid; if (git_blob_create_from_buffer(&blob_oid, git_cas_->GetRepository(), content.c_str(), content.size()) != 0) { std::invoke( *logger, fmt::format("Writing blob into database failed with:\n{}", GitLastError()), /*fatal=*/true); return std::nullopt; } return std::string{git_oid_tostr_s(&blob_oid)}; } catch (std::exception const& ex) { std::invoke(*logger, fmt::format("Write blob failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetObjectByPathFromTree(std::string const& tree_id, std::string const& rel_path) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { std::string entry_id{tree_id}; ObjectType entry_type{ObjectType::Tree}; // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Subtree id retrieval from tree called on a real " "repository"); } // check if path is not trivial if (rel_path != ".") { // get tree object from tree id git_oid tree_oid; if (git_oid_fromstr(&tree_oid, tree_id.c_str()) != 0) { Logger::Log(LogLevel::Trace, "Tree ID parsing in git repository {} failed " "with:\n{}", git_cas_->GetPath().string(), GitLastError()); return std::nullopt; } git_tree* tree_ptr{nullptr}; if (git_tree_lookup( &tree_ptr, git_cas_->GetRepository(), &tree_oid) != 0) { Logger::Log(LogLevel::Trace, "Retrieving tree {} in git repository {} " "failed with:\n{}", tree_id, git_cas_->GetPath().string(), GitLastError()); // cleanup resources git_tree_free(tree_ptr); return std::nullopt; } auto tree = std::unique_ptr( tree_ptr, tree_closer); // get hash for actual entry git_tree_entry* entry_ptr{nullptr}; if (git_tree_entry_bypath( &entry_ptr, tree.get(), rel_path.c_str()) != 0) { Logger::Log(LogLevel::Trace, "Retrieving entry at {} in git repository {} " "failed with:\n{}", rel_path, git_cas_->GetPath().string(), GitLastError()); // cleanup resources git_tree_entry_free(entry_ptr); return std::nullopt; } auto entry = std::unique_ptr( entry_ptr, tree_entry_closer); // get id entry_id = std::string(git_oid_tostr_s(git_tree_entry_id(entry.get()))); // get type if (auto e_type = GitFileModeToObjectType( git_tree_entry_filemode(entry.get()))) { entry_type = *e_type; } else { Logger::Log(LogLevel::Trace, "Retrieving type of entry {} in git repository " "{} failed with:\n{}", entry_id, git_cas_->GetPath().string(), GitLastError()); return std::nullopt; } } // if symlink, also read target if (IsSymlinkObject(entry_type)) { if (auto target = git_cas_->ReadObject(entry_id, /*is_hex_id=*/true)) { return TreeEntryInfo{.id = entry_id, .type = entry_type, .symlink_content = std::move(target)}; } Logger::Log( LogLevel::Trace, "Failed to read target for symlink {} in git repository {}", entry_id, git_cas_->GetPath().string()); return std::nullopt; } return TreeEntryInfo{.id = entry_id, .type = entry_type, .symlink_content = std::nullopt}; } catch (std::exception const& ex) { Logger::Log(LogLevel::Debug, "Get entry by path from tree failed with:\n{}", ex.what()); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::LocalFetchViaTmpRepo(StorageConfig const& storage_config, std::string const& repo_path, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool { #ifdef BOOTSTRAP_BUILD_TOOL return false; #else try { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Branch local fetch called on a real repository"); } auto tmp_dir = storage_config.CreateTypedTmpDir("local_fetch"); if (not tmp_dir) { std::invoke(*logger, "Failed to create temp dir for Git repository", /*fatal=*/true); return false; } auto const& tmp_path = tmp_dir->GetPath(); // create the temporary real repository // it can be bare, as the refspecs for this fetch will be given // explicitly. auto tmp_repo = GitRepo::InitAndOpen(tmp_path, /*is_bare=*/true); if (tmp_repo == std::nullopt) { return false; } // add backend, with max priority FetchIntoODBBackend b{.parent = kFetchIntoODBParent, .target_odb = git_cas_->GetODB()}; if (git_odb_add_backend( tmp_repo->git_cas_->GetODB(), // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) reinterpret_cast(&b), std::numeric_limits::max()) == 0) { // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { std::invoke(*logger, fmt::format("While doing branch local fetch " "via tmp repo:\n{}", msg), fatal); }); // get the config of the correct target repo auto cfg = GetConfigSnapshot(); if (cfg == nullptr) { std::invoke(*logger, fmt::format("Retrieving config object in local " "fetch via tmp repo failed with:\n{}", GitLastError()), true /*fatal*/); return false; } return tmp_repo->FetchFromPath( cfg, repo_path, branch, wrapped_logger); } std::invoke(*logger, fmt::format("Adding custom backend for local fetch failed " "with:\n{}", GitLastError()), true /*fatal*/); return false; } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Fetch {} from local repository {} via tmp dir failed " "with:\n{}", branch ? fmt::format("branch {}", *branch) : "all", repo_path, ex.what()), true /*fatal*/); return false; } #endif // BOOTSTRAP_BUILD_TOOL } auto GitRepo::GetConfigSnapshot() const noexcept -> std::shared_ptr { #ifndef BOOTSTRAP_BUILD_TOOL try { git_config* cfg_ptr{nullptr}; if (git_repository_config_snapshot(&cfg_ptr, git_cas_->GetRepository()) == 0) { return std::shared_ptr(cfg_ptr, config_closer); } } catch (std::exception const& ex) { Logger::Log( LogLevel::Debug, "Unexpected failure getting Git configuration snapshot:\n{}", ex.what()); } #endif // BOOTSTRAP_BUILD_TOOL return nullptr; } auto GitRepo::ImportToGit( StorageConfig const& storage_config, std::filesystem::path const& source_dir, std::string const& commit_message, gsl::not_null const& tagging_lock) noexcept -> expected { // the repository path that imports the content must be separate from the // content path, to avoid polluting the entries auto tmp_dir = storage_config.CreateTypedTmpDir("import_repo"); if (tmp_dir == nullptr) { return unexpected{ std::string("Failed to create tmp path for import repository")}; } // wrap logger for GitRepo call std::string err; auto logger = std::make_shared( [&err](auto const& msg, bool fatal) { if (fatal) { err = msg; } }); auto const& repo_path = tmp_dir->GetPath(); // do the initial commit; no need to guard, as the tmp location is unique auto temp_repo = GitRepo::InitAndOpen(repo_path, /*is_bare=*/false); if (not temp_repo.has_value()) { return unexpected{fmt::format("Could not initialize repository {}", repo_path.string())}; } // stage and commit all err.clear(); auto const commit_hash = temp_repo->CommitDirectory(source_dir, commit_message, logger); if (not commit_hash.has_value()) { return unexpected{ fmt::format("While committing directory {} in repository {}:\n{}", source_dir.string(), repo_path.string(), std::move(err))}; } // open the Git CAS repo auto const just_git_cas = GitCAS::Open(storage_config.GitRoot()); if (just_git_cas == nullptr) { return unexpected{fmt::format("Failed to open Git ODB at {}", storage_config.GitRoot().string())}; } auto just_git_repo = GitRepo::Open(just_git_cas); if (not just_git_repo.has_value()) { return unexpected{fmt::format("Failed to open Git repository {}", storage_config.GitRoot().string())}; } // fetch the new commit into the Git CAS via tmp directory; the call is // thread-safe, so it needs no guarding err.clear(); if (not just_git_repo->LocalFetchViaTmpRepo(storage_config, repo_path.string(), /*branch=*/std::nullopt, logger)) { return unexpected{fmt::format("While fetching in repository {}:\n{}", storage_config.GitRoot().string(), std::move(err))}; } // tag commit and keep it in Git CAS { // this is a non-thread-safe Git operation, so it must be guarded! std::unique_lock slock{*tagging_lock}; // open real repository at Git CAS location auto git_repo = GitRepo::Open(storage_config.GitRoot()); if (not git_repo.has_value()) { return unexpected{ fmt::format("Failed to open Git CAS repository {}", storage_config.GitRoot().string())}; } // Important: message must be consistent with just-mr! err.clear(); if (not git_repo->KeepTag(*commit_hash, /*message=*/"Keep referenced tree alive", logger)) { return unexpected{ fmt::format("While tagging commit {} in repository {}:\n{}", *commit_hash, storage_config.GitRoot().string(), std::move(err))}; } } // get the root tree of this commit; this is thread-safe err.clear(); auto result_tree = just_git_repo->GetSubtreeFromCommit(*commit_hash, ".", logger); if (not result_tree) { return unexpected{ fmt::format("While retrieving tree id of commit {}:\n{}", *commit_hash, std::move(err))}; } return *std::move(result_tree); } auto GitRepo::IsTreeInRepo(std::filesystem::path const& repo, std::string const& tree_id) noexcept -> expected { auto git_cas = GitCAS::Open(repo); if (git_cas == nullptr) { return unexpected( fmt::format("Failed to open Git ODB at {}", repo.string())); } auto git_repo = GitRepo::Open(git_cas); if (not git_repo.has_value()) { return unexpected{ fmt::format("Failed to open Git repository at {}", repo.string())}; } std::string err; auto logger = std::make_shared( [&err](auto const& msg, bool fatal) { if (fatal) { err = msg; } }); auto result = git_repo->CheckTreeExists(tree_id, logger); if (not result.has_value()) { return unexpected{std::move(err)}; } return *result; } auto GitRepo::IsRepoFake() const noexcept -> bool { return is_repo_fake_; } auto GitRepo::ReadDirectTree(std::string const& id, bool is_hex_id, bool ignore_special) const noexcept -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL try { // create object id auto oid = GitObjectID(id, is_hex_id); if (not oid) { return std::nullopt; } // lookup tree git_tree* tree_ptr{nullptr}; if (git_tree_lookup(&tree_ptr, git_cas_->GetRepository(), &(*oid)) != 0) { Logger::Log(LogLevel::Debug, "Failed to lookup Git tree {}", is_hex_id ? std::string{id} : ToHexString(id)); return std::nullopt; } auto tree = std::unique_ptr{ tree_ptr, tree_closer}; // walk tree (flat) and create entries tree_entries_t entries{}; entries.reserve(git_tree_entrycount(tree.get())); if (git_tree_walk(tree.get(), GIT_TREEWALK_PRE, ignore_special ? flat_tree_walker_ignore_special : flat_tree_walker, &entries) != 0) { Logger::Log(LogLevel::Debug, "Failed to walk Git tree {}", is_hex_id ? std::string{id} : ToHexString(id)); return std::nullopt; } #ifndef NDEBUG // Debug-only consistency check for read entries to avoid downstream // failures due to programmatic errors. Expected to always pass. // No need to check if entries exist, so do not pass the Git CAS. EnsuresAudit(ValidateEntries(entries)); #endif return entries; } catch (std::exception const& ex) { Logger::Log( LogLevel::Debug, "Reading direct tree failed with:\n{}", ex.what()); } #endif return std::nullopt; } auto GitRepo::ReadTree(std::string const& id, gsl::not_null const& check_symlinks, bool is_hex_id, bool ignore_special) const noexcept -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL try { auto entries = ReadDirectTree(id, is_hex_id, ignore_special); if (not entries) { return std::nullopt; } // checking non-upwardness of symlinks can not be easily or safely done // during the tree walk, so it is done here. This is only needed for // ignore_special==false. if (not ignore_special) { // we first gather all symlink candidates // to check symlinks in bulk, optimized for network-backed repos std::vector symlinks{}; symlinks.reserve(entries->size()); // at most one symlink per entry for (auto const& entry : *entries) { if (std::any_of(entry.second.begin(), entry.second.end(), [](TreeEntry const& item) { return IsSymlinkObject(item.type); })) { auto digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, ToHexString(entry.first), /*size=*/0, /*is_tree=*/false); if (not digest) { Logger::Log(LogLevel::Debug, "Conversion error in GitRepo:\n {}", std::move(digest).error()); return std::nullopt; } symlinks.emplace_back(*std::move(digest)); } } if (not symlinks.empty() and not std::invoke(check_symlinks.get(), symlinks)) { Logger::Log(LogLevel::Debug, "Found upwards symlinks in Git tree {}", is_hex_id ? std::string{id} : ToHexString(id)); return std::nullopt; } } return entries; } catch (std::exception const& ex) { Logger::Log(LogLevel::Debug, "Reading tree with checker failed with:\n{}", ex.what()); } #endif return std::nullopt; } auto GitRepo::CreateTree(tree_entries_t const& entries) const noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else #ifndef NDEBUG // Check consistency of entries. Also check that entries exist. ExpectsAudit(ValidateEntries(entries, GetGitCAS())); #endif // NDEBUG try { // As the libgit2 treebuilder checks for magic names and does not allow // us to add any and all entries to a Git tree, we resort to // constructing the tree content ourselves and add it manually to the // repository ODB. // We need to sort the filenames according to Git rules: tree entries // need to be considered "as if" their filename has a trailing // separator ('/'). std::map> sorted; for (auto const& [raw_id, es] : entries) { for (auto const& entry : es) { sorted.emplace( entry.name + (IsTreeObject(entry.type) ? "/" : ""), std::make_pair(raw_id, entry.type)); } } // Compute the tree content. For tree entries the trailing slash needs // to be removed from filename before appending it. std::stringstream tree_content{}; for (auto const& [name, entry] : sorted) { std::string_view const filename{ name.data(), name.size() - static_cast(IsTreeObject(entry.second))}; // tree format: " \0[next entries...]" tree_content << fmt::format("{} {}", ObjectTypeToPerm(entry.second), filename) << '\0' << entry.first; } // Write tree to ODB and return raw id string git_oid oid; auto const tree_content_str = tree_content.str(); if (git_odb_write(&oid, git_cas_->GetODB(), tree_content_str.c_str(), tree_content_str.size(), GIT_OBJECT_TREE) != 0) { Logger::Log(LogLevel::Debug, "Failed writing tree to ODB with:\n{}", GitLastError()); return std::nullopt; } return ToRawString(oid); } catch (std::exception const& ex) { Logger::Log( LogLevel::Debug, "Creating tree failed with:\n{}", ex.what()); return std::nullopt; } #endif } auto GitRepo::ReadTreeData( std::string const& data, std::string const& id, gsl::not_null const& check_symlinks, bool is_hex_id) noexcept -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL try { InMemoryODBBackend b{.parent = kInMemoryODBParent}; if (auto raw_id = is_hex_id ? FromHexString(id) : std::make_optional(id)) { try { b.trees.emplace(*raw_id, data); } catch (...) { return std::nullopt; } // create a GitCAS from a special-purpose in-memory object // database. auto cas = GitCAS::CreateEmpty(); if (cas != nullptr and git_odb_add_backend( cas->GetODB(), reinterpret_cast(&b), // NOLINT 0) == 0) { // wrap odb in "fake" repo auto repo = GitRepo(cas); return repo.ReadTree( *raw_id, check_symlinks, /*is_hex_id=*/false); } } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Reading tree data failed with:\n{}", ex.what()); } #endif return std::nullopt; } auto GitRepo::CreateShallowTree(tree_entries_t const& entries) noexcept -> std::optional> { #ifndef BOOTSTRAP_BUILD_TOOL try { InMemoryODBBackend b{.parent = kInMemoryODBParent, .entries = &entries}; auto cas = GitCAS::CreateEmpty(); if (cas != nullptr and git_odb_add_backend( cas->GetODB(), reinterpret_cast(&b), // NOLINT 0) == 0) { // wrap odb in "fake" repo auto repo = GitRepo(cas); if (auto raw_id = repo.CreateTree(entries)) { // read result from in-memory trees if (auto it = b.trees.find(*raw_id); it != b.trees.end()) { return std::make_pair(std::move(*raw_id), std::move(it->second)); } } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Creating shallow tree failed with:\n{}", ex.what()); } #endif return std::nullopt; } [[nodiscard]] auto GitRepo::ReadDirectory( std::filesystem::path const& dir, StoreDirEntryFunc const& read_and_store_entry, anon_logger_ptr const& logger) noexcept -> bool { try { for (auto const& entry : std::filesystem::directory_iterator{dir}) { if (auto type = FileSystemManager::Type(entry.path(), /*allow_upwards=*/true)) { if (not read_and_store_entry( entry.path().filename(), *type, logger)) { // logging with fatal already handled return false; } } else { std::invoke(*logger, fmt::format("Unsupported type for subdir entry {}", entry.path().string()), /*fatal=*/true); return false; } } } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Reading subdirectory {} failed unexpectedly with:\n{}", dir.string(), ex.what()), /*fatal=*/true); return false; } return true; // success! } auto GitRepo::CreateTreeFromDirectory(std::filesystem::path const& dir, anon_logger_ptr const& logger) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else tree_entries_t entries{}; StoreDirEntryFunc dir_read_and_store = [this, &entries, dir](std::filesystem::path const& name, ObjectType type, anon_logger_ptr const& logger) -> bool { try { const auto full_name = dir / name; if (IsTreeObject(type)) { // store subdirectory as a tree in the ODB if (auto raw_id = this->CreateTreeFromDirectory(full_name, logger)) { entries[std::move(*raw_id)].emplace_back(name.string(), ObjectType::Tree); return true; } // logging with fatal already handled return false; } // for non-tree entries, read content and write it as a blob to the // ODB if (auto content = FileSystemManager::ReadContentAtPath(full_name, type)) { if (auto hash = WriteBlob(*content, logger)) { if (auto raw_id = FromHexString(*hash)) { entries[std::move(*raw_id)].emplace_back(name.string(), type); return true; } } } std::invoke( *logger, fmt::format("Failed creating blob {}", full_name.string()), /*fatal=*/true); return false; } catch (std::exception const& ex) { std::invoke( *logger, fmt::format("Unexpectedly failed creating blob with:\n{}", ex.what()), /*fatal=*/true); return false; } }; if (ReadDirectory(dir, dir_read_and_store, logger)) { if (auto tree = CreateTree(entries)) { return tree; } std::invoke( *logger, fmt::format("failed to create tree from entries of directory {}", dir.string()), /*fatal=*/true); } // logging with fatal already handled return std::nullopt; #endif // BOOTSTRAP_BUILD_TOOL } void GitRepo::GitStrArray::AddEntry(std::string entry) { std::size_t const prev_capacity = entries_.capacity(); entries_.emplace_back(std::move(entry)); if (prev_capacity == entries_.capacity()) { entry_pointers_.push_back(entries_.back().data()); } else { // update pointers if reallocation happened entry_pointers_.resize(entries_.size()); for (std::size_t i = 0; i < entries_.size(); ++i) { entry_pointers_[i] = entries_[i].data(); } } } auto GitRepo::GitStrArray::Get() & noexcept -> git_strarray { return git_strarray{.strings = entry_pointers_.data(), .count = entry_pointers_.size()}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_repo.hpp000066400000000000000000000455731516554100600272650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_REPO_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_REPO_HPP #include #include #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" extern "C" { struct git_repository; struct git_config; struct git_strarray; } /// \brief Git repository logic. /// Models both a real repository, owning the underlying ODB /// (non-thread-safe), as well as a "fake" repository, which only wraps an /// existing ODB, allowing thread-safe operations. class GitRepo { public: // Stores the data for defining a single Git tree entry, which consists of // a name (flat basename) and an object type (file/executable/tree). struct TreeEntry { TreeEntry(std::string n, ObjectType t) : name{std::move(n)}, type{t} {} std::string name; ObjectType type; [[nodiscard]] auto operator==(TreeEntry const& other) const noexcept -> bool { return name == other.name and type == other.type; } }; // Tree entries by raw id. The same id might refer to multiple entries. // Note that sharding by id is used as this format enables a more efficient // internal implementation for creating trees. using tree_entries_t = std::unordered_map>; // Stores the info of an object read by its path. struct TreeEntryInfo { std::string id; ObjectType type; // if type is symlink, read it in advance std::optional symlink_content{std::nullopt}; }; // Checks whether a list of symlinks given by their hashes are // non-upwards, based on content read from an actual backend. using SymlinksCheckFunc = std::function const&)>; GitRepo() = delete; // no default ctor ~GitRepo() noexcept = default; // allow only move, no copy GitRepo(GitRepo const&) = delete; GitRepo(GitRepo&&) noexcept; auto operator=(GitRepo const&) = delete; auto operator=(GitRepo&& other) noexcept -> GitRepo&; /// \brief Factory to wrap existing open CAS in a "fake" repository. [[nodiscard]] static auto Open(GitCASPtr git_cas) noexcept -> std::optional; /// \brief Factory to open existing real repository at given location. [[nodiscard]] static auto Open( std::filesystem::path const& repo_path) noexcept -> std::optional; /// \brief Factory to initialize and open new real repository at location. /// Returns nullopt if repository init fails even after repeated tries. [[nodiscard]] static auto InitAndOpen( std::filesystem::path const& repo_path, bool is_bare) noexcept -> std::optional; [[nodiscard]] auto GetGitCAS() const noexcept -> GitCASPtr; [[nodiscard]] auto IsRepoFake() const noexcept -> bool; /// \brief Read entries from tree in CAS. /// Reading a tree must be backed by an object database. Therefore, a real /// repository is required. /// \param id The object id. /// \param is_hex_id Specify whether `id` is hex string or raw. /// \param ignore_special If set, treat symlinks as absent. /// \note This method does not perform any content-based validity checks on /// the read entries. For reading with symlinks validation use ReadTree(). [[nodiscard]] auto ReadDirectTree(std::string const& id, bool is_hex_id = false, bool ignore_special = false) const noexcept -> std::optional; /// \brief Read entries from tree in CAS. /// Reading a tree must be backed by an object database. Therefore, a real /// repository is required. /// \param id The object id. /// \param check_symlinks Function to check non-upwardness condition. /// \param is_hex_id Specify whether `id` is hex string or raw. /// \param ignore_special If set, treat symlinks as absent. [[nodiscard]] auto ReadTree( std::string const& id, gsl::not_null const& check_symlinks, bool is_hex_id = false, bool ignore_special = false) const noexcept -> std::optional; /// \brief Create a flat tree from entries and store tree in CAS. /// Creating a tree must be backed by an object database. Therefore, a real /// repository is required. Furthermore, all entries must be available in /// the underlying object database and object types must correctly reflect /// the type of the object found in the database. /// \param entries The entries to create the tree from. /// \returns The raw object id as string, if successful. [[nodiscard]] auto CreateTree(GitRepo::tree_entries_t const& entries) const noexcept -> std::optional; /// \brief Read entries from tree data (without object db). /// \param data The tree object as plain data. /// \param id The object id. /// \param check_symlinks Function to check non-upwardness condition. /// \param is_hex_id Specify whether `id` is hex string or raw. /// \returns The tree entries. [[nodiscard]] static auto ReadTreeData( std::string const& data, std::string const& id, gsl::not_null const& check_symlinks, bool is_hex_id = false) noexcept -> std::optional; /// \brief Create a flat shallow (without objects in db) tree and return it. /// Creates a tree object from the entries without access to the actual /// blobs. Objects are not required to be available in the underlying object /// database. It is sufficient to provide the raw object id and and object /// type for every entry. /// \param entries The entries to create the tree from. /// \returns A pair of raw object id and the tree object content. [[nodiscard]] static auto CreateShallowTree( GitRepo::tree_entries_t const& entries) noexcept -> std::optional>; using anon_logger_t = std::function; using anon_logger_ptr = std::shared_ptr; /// \brief Create tree from entries at given directory and commit it with /// given message. The given path need not be a subdirectory of the /// repository root path, but the caller must guarantee its entries are /// readable. /// Only possible with real repository and thus non-thread-safe. /// \returns The commit hash, or nullopt if failure. It guarantees the /// logger is called exactly once with fatal if failure. [[nodiscard]] auto CommitDirectory(std::filesystem::path const& dir, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Create annotated tag for given commit. /// Only possible with real repository and thus non-thread-safe. /// Returns the tag on success. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto KeepTag(std::string const& commit, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Retrieves the commit of the HEAD reference. /// Only possible with real repository and thus non-thread-safe. /// Returns the commit hash as a string, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto GetHeadCommit(anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Fetch from given local repository. It can either fetch a given /// named branch, or it can fetch with base refspecs. /// Only possible with real repository and thus non-thread-safe. /// If non-null, use given config snapshot to interact with config entries; /// otherwise, use a snapshot from the current repo and share pointer to it. /// Returns a success flag. It guarantees the logger is called exactly once /// with fatal if failure. [[nodiscard]] auto FetchFromPath(std::shared_ptr cfg, std::string const& repo_path, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool; /// \brief Ensure given tree is kept alive via a tag. It is expected that /// the tree is part of the repository already. /// Only possible with real repository and thus non-thread-safe. /// Returns the tag on success. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto KeepTree(std::string const& tree_id, std::string const& message, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Get the tree id of a subtree given the root commit /// Calling it from a fake repository allows thread-safe use. /// Returns the subtree hash on success or an unexpected error. [[nodiscard]] auto GetSubtreeFromCommit( std::string const& commit, std::string const& subdir, anon_logger_ptr const& logger) noexcept -> expected; /// \brief Get the tree id of a subtree given the root tree hash /// Calling it from a fake repository allows thread-safe use. /// Returns the subtree hash, as a string, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto GetSubtreeFromTree( std::string const& tree_id, std::string const& subdir, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Get the tree id of a subtree given a filesystem directory path. /// Calling it from a fake repository allows thread-safe use. /// Will search for the root of the repository containing the path, /// and deduce the subdir relationship in the Git tree. /// Requires for the HEAD commit to be already known. /// Returns the tree hash, as a string, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto GetSubtreeFromPath( std::filesystem::path const& fpath, std::string const& head_commit, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Check if given commit is part of the local repository. /// Calling it from a fake repository allows thread-safe use. /// Returns a status of commit presence, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto CheckCommitExists(std::string const& commit, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Try to retrieve the root of the repository containing the /// given path, if the path is actually part of a repository. /// Returns the git folder if path is in a git repo, empty string if path is /// not in a git repo, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GetRepoRootFromPath( std::filesystem::path const& fpath, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Check if given tree ID is present in the directory structure of /// the local repository. /// Calling it from a fake repository allows thread-safe use. /// Returns a status of tree presence, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto CheckTreeExists(std::string const& tree_id, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Check if given blob ID is present in the directory structure of /// the local repository. /// Calling it from a fake repository allows thread-safe use. /// Returns a status of blob presence, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto CheckBlobExists(std::string const& blob_id, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Check if given blob ID is present in the directory structure /// of the local repository and try to return it. Calling it from a fake /// repository allows thread-safe use. Returns a pair of a success flag, /// stating that no errors occurred while performing the libgit2 calls, /// and an optional string containing the content of the blob, if the /// blob is found. It guarantees the logger is called exactly once with /// fatal if failure. [[nodiscard]] auto TryReadBlob(std::string const& blob_id, anon_logger_ptr const& logger) noexcept -> std::pair>; /// \brief Write given content as a blob into underlying object database. /// Calling it from a fake repository allows thread-safe use. /// \returns Git ID of the written blob, or nullopt on errors. /// It guarantees the logger is called exactly once with fatal if failure. /// Use with care, especially for large objects. [[nodiscard]] auto WriteBlob(std::string const& content, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Get the object info related to a given path inside a Git tree. /// Unlike GetSubtreeFromTree, we here ignore errors and only return a value /// when all is successful. /// Calling it from a fake repository allows thread-safe use. [[nodiscard]] auto GetObjectByPathFromTree( std::string const& tree_id, std::string const& rel_path) noexcept -> std::optional; /// \brief Fetch from given local repository via a temporary location. Uses /// tmp dir to fetch asynchronously using libgit2. /// Uses either a given branch, or fetches all (with base refspecs). /// Returns a success flag. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto LocalFetchViaTmpRepo( StorageConfig const& storage_config, std::string const& repo_path, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool; /// \brief Get a snapshot of the repository configuration. /// Returns nullptr on errors. [[nodiscard]] auto GetConfigSnapshot() const noexcept -> std::shared_ptr; /// \brief Import source directory to target git repository /// \param storage_config Storage where the source must be imported to /// \param source_dir Directory to import /// \param commit_message Message of the commit /// \param tagging_lock Mutex to protect critical git operations /// \return The tree id of the committed directory on success or an /// unexpected error as string on failure. [[nodiscard]] static auto ImportToGit( StorageConfig const& storage_config, std::filesystem::path const& source_dir, std::string const& commit_message, gsl::not_null const& tagging_lock) noexcept -> expected; /// \brief Check that the given repository contains the given tree /// \returns Flag reflecting whether the tree is present in the repository /// or an error message on failure. [[nodiscard]] static auto IsTreeInRepo(std::filesystem::path const& repo, std::string const& tree_id) noexcept -> expected; private: GitCASPtr git_cas_; // default to real repo, as that is non-thread-safe bool is_repo_fake_; protected: /// \brief Open "fake" repository wrapper for existing CAS. explicit GitRepo(GitCASPtr git_cas) noexcept; /// \brief Open real repository at given location. explicit GitRepo(std::filesystem::path const& repo_path) noexcept; /// \brief Function type handling directory entries read from filesystem. /// \returns Success flag. Must guarantee that the logger is called exactly /// once with fatal if returning false. using StoreDirEntryFunc = std::function< bool(std::filesystem::path const&, ObjectType, anon_logger_ptr const&)>; /// \brief Helper function to read the entries of a filesystem subdirectory /// and store them to the ODB. It is a modified version of the same-named /// function from FileSystemManager which accepts a subdir and a specific /// logger instead of the default. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto ReadDirectory( std::filesystem::path const& dir, StoreDirEntryFunc const& read_and_store_entry, anon_logger_ptr const& logger) noexcept -> bool; /// \brief Create a tree from the content of a directory by recursively /// adding its entries to the object database. /// \return The raw id of the tree. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto CreateTreeFromDirectory( std::filesystem::path const& dir, anon_logger_ptr const& logger) noexcept -> std::optional; class GitStrArray final { public: void AddEntry(std::string entry); [[nodiscard]] auto Get() & noexcept -> git_strarray; private: std::vector entries_; std::vector entry_pointers_; }; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_REPO_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_tree.cpp000066400000000000000000000143271516554100600272430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_tree.hpp" #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" namespace { // resolve '.' and '..' in path. [[nodiscard]] auto ResolveRelativePath( std::filesystem::path const& path) noexcept -> std::filesystem::path { auto normalized = path.lexically_normal(); return (normalized / "").parent_path(); // strip trailing slash } [[nodiscard]] auto LookupEntryPyPath( GitTree const& tree, std::filesystem::path::const_iterator it, std::filesystem::path::const_iterator const& end, bool ignore_special = false) noexcept -> GitTreeEntryPtr { auto segment = *it; auto entry = tree.LookupEntryByName(segment); if (not entry) { return nullptr; } if (++it != end) { auto const& subtree = entry->Tree(ignore_special); if (not subtree) { return nullptr; } return LookupEntryPyPath(*subtree, it, end); } return entry; } class SymlinksChecker final { public: explicit SymlinksChecker(gsl::not_null const& cas) noexcept : cas_{*cas} {} [[nodiscard]] auto operator()( std::vector const& ids) const noexcept -> bool { return std::all_of( ids.begin(), ids.end(), [&cas = cas_](ArtifactDigest const& id) { return cas .ReadObject(id.hash(), /*is_hex_id=*/true, /*as_valid_symlink=*/true) .has_value(); }); }; private: GitCAS const& cas_; }; } // namespace auto GitTree::Read(std::filesystem::path const& repo_path, std::string const& tree_id) noexcept -> std::optional { auto cas = GitCAS::Open(repo_path); if (not cas) { return std::nullopt; } return Read(cas, tree_id); } auto GitTree::Read(gsl::not_null const& cas, std::string const& tree_id, bool ignore_special, bool skip_checks) noexcept -> std::optional { if (auto raw_id = FromHexString(tree_id)) { auto repo = GitRepo::Open(cas); if (repo != std::nullopt) { auto entries = skip_checks ? repo->ReadDirectTree( *raw_id, /*is_hex_id=*/false, ignore_special) : repo->ReadTree(*raw_id, SymlinksChecker{cas}, /*is_hex_id=*/false, ignore_special); if (entries) { // NOTE: the raw_id value is NOT recomputed when // ignore_special==true. return GitTree::FromEntries( cas, std::move(*entries), *raw_id, ignore_special); } } else { return ::std::nullopt; } } return std::nullopt; } auto GitTree::LookupEntryByName(std::string const& name) const noexcept -> GitTreeEntryPtr { auto entry_it = entries_.find(name); if (entry_it == entries_.end()) { Logger::Log( LogLevel::Debug, "git tree does not contain entry {}", name); return nullptr; } return entry_it->second; } auto GitTree::LookupEntryByPath( std::filesystem::path const& path) const noexcept -> GitTreeEntryPtr { auto resolved = ResolveRelativePath(path); return LookupEntryPyPath( *this, resolved.begin(), resolved.end(), ignore_special_); } auto GitTree::Size() const noexcept -> std::optional { if (auto header = cas_->ReadHeader(raw_id_, /*is_hex_id=*/false)) { return header->first; } return std::nullopt; } auto GitTree::RawData() const noexcept -> std::optional { return cas_->ReadObject(raw_id_, /*is_hex_id=*/false); } auto GitTreeEntry::Blob() const noexcept -> std::optional { if (not IsBlob()) { return std::nullopt; } // return only valid blobs return cas_->ReadObject(raw_id_, /*is_hex_id=*/false, /*as_valid_symlink=*/IsSymlinkObject(type_)); } auto GitTreeEntry::Tree(bool ignore_special) const& noexcept -> std::optional const& { return tree_cached_.SetOnceAndGet( [this, ignore_special]() -> std::optional { if (IsTree()) { auto repo = GitRepo::Open(cas_); if (repo == std::nullopt) { return std::nullopt; } auto entries = repo->ReadTree(raw_id_, SymlinksChecker{cas_}, /*is_hex_id=*/false, ignore_special); if (entries) { // NOTE: the raw_id value is NOT recomputed when // ignore_special==true. return GitTree::FromEntries( cas_, std::move(*entries), raw_id_, ignore_special); } } return std::nullopt; }); } auto GitTreeEntry::Size() const noexcept -> std::optional { if (auto header = cas_->ReadHeader(raw_id_, /*is_hex_id=*/false)) { return header->first; } return std::nullopt; } auto GitTreeEntry::RawData() const noexcept -> std::optional { return cas_->ReadObject(raw_id_, /*is_hex_id=*/false); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_tree.hpp000066400000000000000000000156101516554100600272440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP #include #include #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/atomic_value.hpp" #include "src/utils/cpp/hex_string.hpp" class GitTreeEntry; using GitTreeEntryPtr = std::shared_ptr; class GitTree { friend class GitTreeEntry; public: using entries_t = std::unordered_map>; /// \brief Read tree with given id from Git repository. /// \param repo_path Path to the Git repository. /// \param tree_id Tree id as as hex string. [[nodiscard]] static auto Read(std::filesystem::path const& repo_path, std::string const& tree_id) noexcept -> std::optional; /// \brief Read tree with given id from CAS. /// \param cas Git CAS that contains the tree id. /// \param tree_id Tree id as as hex string. /// \param ignore_special If set, treat symlinks as absent. /// \param skip_checks If set, skip any symlinks checks. /// NOTE: If ignore_special==true, the stored entries might differ from the /// actual tree, as some filesystem entries get skipped. [[nodiscard]] static auto Read(gsl::not_null const& cas, std::string const& tree_id, bool ignore_special = false, bool skip_checks = false) noexcept -> std::optional; /// \brief Lookup by dir entry name. '.' and '..' are not allowed. [[nodiscard]] auto LookupEntryByName(std::string const& name) const noexcept -> GitTreeEntryPtr; /// \brief Lookup by relative path. '.' is not allowed. [[nodiscard]] auto LookupEntryByPath( std::filesystem::path const& path) const noexcept -> GitTreeEntryPtr; [[nodiscard]] auto begin() const noexcept { return entries_.begin(); } [[nodiscard]] auto end() const noexcept { return entries_.end(); } // Getter for the root tree id with fast lookup flag check. This enforces // automatically that no filesystem entry was skipped during creation. [[nodiscard]] auto Hash() const noexcept { return ignore_special_ ? std::nullopt : std::make_optional(ToHexString(raw_id_)); } // Getter of the hex root tree id with no fast lookup flag check. As such, // the caller MUST NOT assume that there is a one-to-one correspondence // between this returned tree id and the values stored in entries_. [[nodiscard]] auto FileRootHash() const noexcept { return ToHexString(raw_id_); } // Getter of the raw root tree id with no fast lookup flag check. As such, // the caller MUST NOT assume that there is a one-to-one correspondence // between this returned tree id and the values stored in entries_. [[nodiscard]] auto RawFileRootHash() const noexcept { return raw_id_; } [[nodiscard]] auto Size() const noexcept -> std::optional; [[nodiscard]] auto RawData() const noexcept -> std::optional; private: gsl::not_null cas_; // If not ignore_special_, contains all the entries of tree raw_id_. entries_t entries_; // Stores the root id of the tree; if ignore_special_, this is not // guaranteed to be the same as the id of the tree containing entries_. std::string raw_id_; // If set, ignore all fast tree lookups and always traverse. bool ignore_special_; explicit GitTree(gsl::not_null const& cas, entries_t&& entries, std::string raw_id, bool ignore_special = false) noexcept : cas_{cas}, entries_{std::move(entries)}, raw_id_{std::move(raw_id)}, ignore_special_{ignore_special} {} [[nodiscard]] static auto FromEntries(gsl::not_null const& cas, GitRepo::tree_entries_t&& entries, std::string raw_id, bool ignore_special = false) noexcept -> std::optional { entries_t e{}; e.reserve(entries.size()); for (auto& [id, es] : entries) { for (auto& entry : es) { try { e.emplace( std::move(entry.name), std::make_shared(cas, id, entry.type)); } catch (...) { return std::nullopt; } } } return GitTree(cas, std::move(e), std::move(raw_id), ignore_special); } }; class GitTreeEntry { public: GitTreeEntry(gsl::not_null const& cas, std::string raw_id, ObjectType type) noexcept : cas_{cas}, raw_id_{std::move(raw_id)}, type_{type} {} [[nodiscard]] auto IsBlob() const noexcept { return IsFileObject(type_) or IsSymlinkObject(type_); } [[nodiscard]] auto IsTree() const noexcept { return IsTreeObject(type_); } [[nodiscard]] auto Blob() const noexcept -> std::optional; [[nodiscard]] auto Tree(bool ignore_special = false) const& noexcept -> std::optional const&; [[nodiscard]] auto Hash() const noexcept { return ToHexString(raw_id_); } [[nodiscard]] auto Type() const noexcept { return type_; } // Use with care. Implementation might read entire object to obtain // size. Consider using Blob()->size() instead. [[nodiscard]] auto Size() const noexcept -> std::optional; [[nodiscard]] auto RawData() const noexcept -> std::optional; private: gsl::not_null cas_; std::string raw_id_; ObjectType type_; AtomicValue> tree_cached_; }; using GitTreePtr = std::shared_ptr; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_tree_utils.cpp000066400000000000000000000106361516554100600304620ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_tree_utils.hpp" #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/fs_utils.hpp" namespace { /// \brief Mark a Git hash as corresponding to a valid tree by creating a /// corresponding marker file. /// \returns Success flag. [[nodiscard]] auto MarkTreeValid(StorageConfig const& storage_config, std::string const& tree_id) noexcept -> bool { auto const marker = StorageUtils::GetValidTreesMarkerFile(storage_config, tree_id); return FileSystemManager::CreateDirectory(marker.parent_path()) and FileSystemManager::CreateFile(marker); } /// \brief Checks if a given Git hash is known to correspond to a validated /// tree by checking the existence of its respective marker file. /// \returns Existence flag signaling validation. [[nodiscard]] auto IsTreeValid(StorageConfig const& storage_config, std::string const& tree_hash) noexcept -> bool { // check in all generations for (std::size_t generation = 0; generation < storage_config.num_generations; ++generation) { if (FileSystemManager::Exists(StorageUtils::GetValidTreesMarkerFile( storage_config, tree_hash, generation))) { // ensure it is marked in current generation return generation == 0 ? true : MarkTreeValid(storage_config, tree_hash); } } return false; } /// \brief Validate a GitTree's subtrees recursively. /// \returns True if all the subtrees are valid. [[nodiscard]] auto ValidateGitSubtrees(StorageConfig const& storage_config, GitTree const& tree) noexcept -> bool { for (auto const& [path, entry] : tree) { if (IsTreeObject(entry->Type())) { auto const hash = entry->Hash(); if (not IsTreeValid(storage_config, hash)) { // validate subtree auto subtree = entry->Tree(); if (not subtree or not ValidateGitSubtrees(storage_config, *subtree) or not MarkTreeValid(storage_config, hash)) { return false; } } } } return true; } } // namespace namespace GitTreeUtils { auto ReadValidGitCASTree(StorageConfig const& storage_config, std::string const& tree_id, GitCASPtr const& git_cas) noexcept -> std::optional { if (IsTreeValid(storage_config, tree_id)) { // read tree without extra checks return GitTree::Read( git_cas, tree_id, /*ignore_special=*/false, /*skip_checks=*/true); } // read GitTree from Git with checks and validate its subtrees recursively if (auto tree = GitTree::Read(git_cas, tree_id)) { if (ValidateGitSubtrees(storage_config, *tree) and MarkTreeValid(storage_config, tree_id)) { return tree; } } return std::nullopt; } auto IsGitTreeValid(StorageConfig const& storage_config, GitTreeEntryPtr const& entry) noexcept -> bool { if (entry == nullptr) { return false; } auto tree_id = entry->Hash(); if (IsTreeValid(storage_config, tree_id)) { return true; } // read underlying GitTree and validate its subtrees recursively if (auto const& read_tree = entry->Tree()) { if (ValidateGitSubtrees(storage_config, *read_tree) and MarkTreeValid(storage_config, tree_id)) { return true; } } return false; } } // namespace GitTreeUtils just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_tree_utils.hpp000066400000000000000000000050241516554100600304620ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_UTILS_HPP #include #include #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/buildtool/storage/config.hpp" /// \brief Utility methods for validating GitTree instances namespace GitTreeUtils { /// \brief Read a GitTree from a Git repository and ensure (recursively) that it /// is free of upwards symlinks. Performs storage-based caching of all found /// valid tree hashes. /// \param storage_config Storage instance for caching valid tree hashes. /// \param tree_id Git identifier of the tree to read and validate. /// \param git_cas Git repository providing the tree. /// \returns GitTree instance free of upwards symlinks, recursively, on success /// or nullopt on failure. [[nodiscard]] auto ReadValidGitCASTree(StorageConfig const& storage_config, std::string const& tree_id, GitCASPtr const& git_cas) noexcept -> std::optional; /// \brief Validate a known GitTreeEntry pointing to a Git tree, by checking /// recursively that it is free of upwards symlinks. Performs storage-based /// caching of all found valid tree hashes. /// \param storage_config Storage instance for caching valid tree hashes. /// \param GitTreeEntryPtr Pointer to an existing GitTreeEntry. /// \returns Flag stating if tree is (recursively) free of upwards symlinks. /// \note This method is useful when one has fast (and preferably cached) access /// to a GitTree instance and direct reading from a repository is not desired. [[nodiscard]] auto IsGitTreeValid(StorageConfig const& storage_config, GitTreeEntryPtr const& entry) noexcept -> bool; } // namespace GitTreeUtils #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_types.hpp000066400000000000000000000016001516554100600274430ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TYPES_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TYPES_HPP #include enum class GitLookupError : std::uint8_t { Fatal = 0, NotFound = 1, }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TYPES_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_utils.cpp000066400000000000000000000065501516554100600274430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_utils.hpp" #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" extern "C" { #include } #ifndef BOOTSTRAP_BUILD_TOOL namespace { constexpr std::size_t kOIDRawSize{GIT_OID_RAWSZ}; constexpr std::size_t kOIDHexSize{GIT_OID_HEXSZ}; } // namespace #endif // BOOTSTRAP_BUILD_TOOL auto GitLastError() noexcept -> std::string { #ifndef BOOTSTRAP_BUILD_TOOL git_error const* const err = git_error_last(); if (err != nullptr and err->message != nullptr) { return fmt::format("error code {}: {}", err->klass, err->message); } #endif // BOOTSTRAP_BUILD_TOOL return ""; } auto GitObjectID(std::string const& id, bool is_hex_id) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else if (id.size() < (is_hex_id ? kOIDHexSize : kOIDRawSize)) { Logger::Log(LogLevel::Error, "invalid git object id {}", is_hex_id ? id : ToHexString(id)); return std::nullopt; } git_oid oid{}; if (is_hex_id and git_oid_fromstr(&oid, id.c_str()) == 0) { return oid; } if (not is_hex_id and git_oid_fromraw( &oid, reinterpret_cast(id.data()) // NOLINT ) == 0) { return oid; } Logger::Log(LogLevel::Error, "parsing git object id {} failed with:\n{}", is_hex_id ? id : ToHexString(id), GitLastError()); return std::nullopt; #endif // BOOTSTRAP_BUILD_TOOL } void odb_closer(gsl::owner odb) { #ifndef BOOTSTRAP_BUILD_TOOL git_odb_free(odb); #endif } void repository_closer(gsl::owner repository) { #ifndef BOOTSTRAP_BUILD_TOOL git_repository_free(repository); #endif } void tree_closer(gsl::owner tree) { #ifndef BOOTSTRAP_BUILD_TOOL git_tree_free(tree); #endif } void signature_closer(gsl::owner signature) { #ifndef BOOTSTRAP_BUILD_TOOL git_signature_free(signature); #endif } void object_closer(gsl::owner object) { #ifndef BOOTSTRAP_BUILD_TOOL git_object_free(object); #endif } void remote_closer(gsl::owner remote) { #ifndef BOOTSTRAP_BUILD_TOOL git_remote_free(remote); #endif } void commit_closer(gsl::owner commit) { #ifndef BOOTSTRAP_BUILD_TOOL git_commit_free(commit); #endif } void tree_entry_closer(gsl::owner tree_entry) { #ifndef BOOTSTRAP_BUILD_TOOL git_tree_entry_free(tree_entry); #endif } void config_closer(gsl::owner cfg) { #ifndef BOOTSTRAP_BUILD_TOOL git_config_free(cfg); #endif } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/git_utils.hpp000066400000000000000000000037301516554100600274450ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_UTILS_HPP #include #include #include #include "gsl/gsl" extern "C" { struct git_oid; struct git_odb; struct git_repository; struct git_tree; struct git_signature; struct git_object; struct git_remote; struct git_commit; struct git_tree_entry; struct git_config; } // time in ms between tries for git locks constexpr std::size_t kGitLockWaitTime{100}; // number of retries for git locks constexpr std::size_t kGitLockNumTries{10}; [[nodiscard]] auto GitObjectID(std::string const& id, bool is_hex_id = false) noexcept -> std::optional; /// \brief Retrieve error message of last libgit2 call. [[nodiscard]] auto GitLastError() noexcept -> std::string; void odb_closer(gsl::owner odb); void repository_closer(gsl::owner repository); void tree_closer(gsl::owner tree); void signature_closer(gsl::owner signature); void object_closer(gsl::owner object); void remote_closer(gsl::owner remote); void commit_closer(gsl::owner commit); void tree_entry_closer(gsl::owner tree_entry); void config_closer(gsl::owner cfg); #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/jsonfs.hpp000066400000000000000000000045401516554100600267440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" class Json { public: // Note that we are not using std::pair and being // coherent with FileSystemManager::ReadFile because there is a bug in llvm // toolchain related to type_traits that does not allow us to use // std::pair where T or U are nlohmann::json. // LLVM bug report: https://bugs.llvm.org/show_bug.cgi?id=48507 // Minimal example: https://godbolt.org/z/zacedsGzo [[nodiscard]] static auto ReadFile( std::filesystem::path const& file) noexcept -> std::optional { auto const type = FileSystemManager::Type(file); if (not type or not IsFileObject(*type)) { Logger::Log(LogLevel::Debug, "{} can not be read because it is not a file.", file.string()); return std::nullopt; } try { nlohmann::json content; std::ifstream file_reader(file.string()); if (file_reader.is_open()) { file_reader >> content; return content; } return std::nullopt; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, e.what()); return std::nullopt; } } }; // Class Json #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/object_cas.hpp000066400000000000000000000170701516554100600275400ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP #include #include #include #include #include // std::move #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" /// \brief CAS for storing objects as plain blobs. /// Automatically computes the digest for storing a blob from file path/bytes. /// The actual object type is given as a template parameter. Depending on the /// object type, files written to the file system may have different properties /// (e.g., the x-bit set) or the digest may be computed differently (e.g., tree /// digests in non-compatible mode). Supports custom "exists callback", which /// is used to check blob existence before every read and write operation. /// \tparam kType The object type to store as blob. template class ObjectCAS { public: /// \brief Callback type for checking blob existence. /// \returns true if a blob for the given digest exists at the given path. using ExistsFunc = std::function; /// \brief Create new object CAS in store_path directory. /// The optional "exists callback" is used to check blob existence before /// every read and write operation. It promises that a blob for the given /// digest exists at the given path if true was returned. /// \param store_path The path to use for storing blobs. /// \param exists (optional) Function for checking blob existence. explicit ObjectCAS( HashFunction hash_function, std::filesystem::path const& store_path, std::optional> exists = std::nullopt) : file_store_{store_path}, exists_{exists.has_value() ? std::move(exists)->get() : kDefaultExists}, hash_function_{hash_function} {} ObjectCAS(ObjectCAS const&) = delete; ObjectCAS(ObjectCAS&&) = delete; auto operator=(ObjectCAS const&) -> ObjectCAS& = delete; auto operator=(ObjectCAS&&) -> ObjectCAS& = delete; ~ObjectCAS() noexcept = default; /// \brief Obtain path to the storage root. [[nodiscard]] auto StorageRoot() const noexcept -> std::filesystem::path const& { return file_store_.StorageRoot(); } /// \brief Store blob from bytes. /// \param bytes The bytes do create the blob from. /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromBytes(std::string const& bytes) const noexcept -> std::optional { return StoreBlob(bytes, /*is_owner=*/true); } /// \brief Store blob from file path. /// \param file_path The path of the file to store as blob. /// \param is_owner Indicates ownership for optimization (hardlink). /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromFile(std::filesystem::path const& file_path, bool is_owner = false) const noexcept -> std::optional { return StoreBlob(file_path, is_owner); } /// \brief Get path to blob. /// \param digest Digest of the blob to lookup. /// \returns Path to blob if found or nullopt otherwise. [[nodiscard]] auto BlobPath(ArtifactDigest const& digest) const noexcept -> std::optional { auto const& id = digest.hash(); auto blob_path = file_store_.GetPath(id); if (not IsAvailable(digest, blob_path)) { logger_.Emit(LogLevel::Trace, "Blob not found {}", id); return std::nullopt; } return blob_path; } private: // For `Tree` the underlying storage type is non-executable file. static constexpr ObjectType kStorageType = IsTreeObject(kType) ? ObjectType::File : kType; Logger logger_{std::string{"ObjectCAS"} + ToChar(kType)}; FileStorage file_store_; gsl::not_null exists_; HashFunction hash_function_; /// Default callback for checking blob existence. static inline ExistsFunc const kDefaultExists = [](auto const& /*digest*/, auto const& path) { return FileSystemManager::IsFile(path); }; [[nodiscard]] auto CreateDigest(std::string const& bytes) const noexcept -> std::optional { return ArtifactDigestFactory::HashDataAs(hash_function_, bytes); } [[nodiscard]] auto CreateDigest(std::filesystem::path const& file_path) const noexcept -> std::optional { return ArtifactDigestFactory::HashFileAs(hash_function_, file_path); } [[nodiscard]] auto IsAvailable( ArtifactDigest const& digest, std::filesystem::path const& path) const noexcept -> bool { try { return std::invoke(exists_.get(), digest, path); } catch (...) { return false; } } /// \brief Store blob from bytes to storage. [[nodiscard]] auto StoreBlobData(std::string const& blob_id, std::string const& bytes, bool /*unused*/) const noexcept -> bool { return file_store_.AddFromBytes(blob_id, bytes); } /// \brief Store blob from file path to storage. [[nodiscard]] auto StoreBlobData(std::string const& blob_id, std::filesystem::path const& file_path, bool is_owner) const noexcept -> bool { return file_store_.AddFromFile(blob_id, file_path, is_owner); } /// \brief Store blob from unspecified data to storage. template [[nodiscard]] auto StoreBlob(T const& data, bool is_owner) const noexcept -> std::optional { if (auto digest = CreateDigest(data)) { auto const& id = digest->hash(); if (IsAvailable(*digest, file_store_.GetPath(id))) { return digest; } if (StoreBlobData(id, data, is_owner)) { return digest; } logger_.Emit(LogLevel::Debug, "Failed to store blob {}.", id); } logger_.Emit(LogLevel::Debug, "Failed to create digest."); return std::nullopt; } }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/object_type.hpp000066400000000000000000000054201516554100600277470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP #include #include "gsl/gsl" enum class ObjectType : std::int8_t { File, Executable, Tree, Symlink // non-upwards symbolic link }; [[nodiscard]] constexpr auto FromChar(char c) noexcept -> ObjectType { switch (c) { case 'x': return ObjectType::Executable; case 't': return ObjectType::Tree; case 'l': return ObjectType::Symlink; default: return ObjectType::File; } Ensures(false); // unreachable } [[nodiscard]] constexpr auto ToChar(ObjectType type) noexcept -> char { switch (type) { case ObjectType::File: return 'f'; case ObjectType::Executable: return 'x'; case ObjectType::Tree: return 't'; case ObjectType::Symlink: return 'l'; } Ensures(false); // unreachable } [[nodiscard]] constexpr auto IsFileObject(ObjectType type) noexcept -> bool { return type == ObjectType::Executable or type == ObjectType::File; } [[nodiscard]] constexpr auto IsExecutableObject(ObjectType type) noexcept -> bool { return type == ObjectType::Executable; } [[nodiscard]] constexpr auto IsTreeObject(ObjectType type) noexcept -> bool { return type == ObjectType::Tree; } /// \brief Non-upwards symlinks are designated as first-class citizens. [[nodiscard]] constexpr auto IsSymlinkObject(ObjectType type) noexcept -> bool { return type == ObjectType::Symlink; } /// \brief Valid blob sources can be files, executables, or symlinks. [[nodiscard]] constexpr auto IsBlobObject(ObjectType type) noexcept -> bool { return type == ObjectType::Executable or type == ObjectType::File or type == ObjectType::Symlink; } /// \brief Only regular files, executables, and trees are non-special entries. [[nodiscard]] constexpr auto IsNonSpecialObject(ObjectType type) noexcept -> bool { return type == ObjectType::File or type == ObjectType::Executable or type == ObjectType::Tree; } #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/precomputed_root.cpp000066400000000000000000000227521516554100600310340ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/precomputed_root.hpp" #include #include #include "fmt/core.h" #include "gsl/gsl" #include "src/utils/cpp/hash_combine.hpp" namespace { template [[nodiscard]] auto ParseImpl(nlohmann::json const& /*root*/) -> expected { Ensures(false); } template [[nodiscard]] auto ParsePrecomputed(nlohmann::json const& root) noexcept -> expected { try { auto parsed_root = ParseImpl(root); if (not parsed_root) { return unexpected{fmt::format("While parsing {} root {}:\n{}", T::kMarker, root.dump(), std::move(parsed_root).error())}; } return PrecomputedRoot{*std::move(parsed_root)}; } catch (const std::exception& e) { return unexpected{fmt::format("While parsing {} root {}:\n{}", T::kMarker, root.dump(), e.what())}; } } template <> [[nodiscard]] auto ParseImpl(nlohmann::json const& root) -> expected { if ((root.size() != ComputedRoot::kSchemeLength) and (root.size() != ComputedRoot::kSchemeLength + 1)) { return unexpected{fmt::format( "The root has a wrong number of arguments: {}\nThe scheme requires " "[, , , , ] optionally " "followed by a pragma object", root.dump())}; } if (not root[1].is_string()) { return unexpected{fmt::format( "The root has a wrong type of . Expected a string, got {}", root[1].dump())}; } if (not root[2].is_string()) { return unexpected{fmt::format( "The root has a wrong type of . Expected a string, got {}", root[2].dump())}; } if (not root[3].is_string()) { return unexpected{fmt::format( "The root has a wrong type of . Expected a string, got {}", root[3].dump())}; } if (not root[4].is_object()) { return unexpected{ fmt::format("The root has a wrong type of . Expected a " "plain json object, got {}", root[4].dump())}; } auto absent = false; if (root.size() > ComputedRoot::kSchemeLength) { if (not root[ComputedRoot::kSchemeLength].is_object()) { return unexpected{ fmt::format("The root has a wrong type of optional pragma " "argument. Expected a " "plain json object, got {}", root[ComputedRoot::kSchemeLength].dump())}; } if (root[ComputedRoot::kSchemeLength].contains("absent")) { auto absent_entry = root[ComputedRoot::kSchemeLength]["absent"]; if (not absent_entry.is_boolean()) { return unexpected{ fmt::format("Expected pragma entry \"absent\" to be " "boolean, but found {}", absent_entry.dump())}; } absent = absent_entry; } } return ComputedRoot{.repository = std::string{root[1]}, .target_module = std::string{root[2]}, .target_name = std::string{root[3]}, .config = root[4], .absent = absent}; } template <> [[nodiscard]] auto ParseImpl(nlohmann::json const& root) -> expected { if (root.size() != TreeStructureRoot::kSchemeLength and root.size() != TreeStructureRoot::kSchemePragmaLength) { return unexpected{ fmt::format("The root has a wrong number of arguments: {}\nThe " "scheme requires [, ] optionally " "followed by a pragma object", root.dump())}; } if (not root[1].is_string()) { return unexpected{fmt::format( "The root has a wrong type of . Expected a string, got {}", root[1].dump())}; } bool absent = false; if (root.size() == TreeStructureRoot::kSchemePragmaLength) { if (not root[2].is_object()) { return unexpected{ fmt::format("The root has a wrong type of optional pragma " "argument. Expected a plain json object, got {}", root[2].dump())}; } if (root[2].contains("absent")) { auto const& absent_entry = root[2].at("absent"); if (not absent_entry.is_null() and not absent_entry.is_boolean()) { return unexpected{ fmt::format("Expected pragma entry \"absent\" to be " "boolean, but found {}", absent_entry.dump())}; } absent = absent_entry.is_boolean() and absent_entry.template get(); } } return TreeStructureRoot{.repository = root[1].template get(), .absent = absent}; } } // namespace auto ComputedRoot::operator==(ComputedRoot const& other) const noexcept -> bool { return repository == other.repository and target_module == other.target_module and target_name == other.target_name and config == other.config and absent == other.absent; } auto ComputedRoot::operator<(ComputedRoot const& other) const noexcept -> bool { if (auto const res = repository <=> other.repository; res != 0) { return res < 0; } if (auto const res = target_module <=> other.target_module; res != 0) { return res < 0; } if (auto const res = target_name <=> other.target_name; res != 0) { return res < 0; } if (absent != other.absent) { return absent; } return config < other.config; } auto ComputedRoot::ToString() const -> std::string { return fmt::format("([\"@\", {}, {}, {}{}], {})", nlohmann::json(repository).dump(), nlohmann::json(target_module).dump(), nlohmann::json(target_name).dump(), absent ? ", {\"absent\": true}" : "", config.dump()); } auto ComputedRoot::ComputeHash() const -> std::size_t { size_t seed{}; hash_combine(&seed, kMarker); hash_combine(&seed, repository); hash_combine(&seed, target_module); hash_combine(&seed, target_name); hash_combine(&seed, absent); hash_combine(&seed, config); return seed; } auto TreeStructureRoot::operator==( TreeStructureRoot const& other) const noexcept -> bool { return absent == other.absent and repository == other.repository; } auto TreeStructureRoot::operator<(TreeStructureRoot const& other) const noexcept -> bool { if (absent != other.absent) { return absent; } return repository < other.repository; } auto TreeStructureRoot::ToString() const -> std::string { return fmt::format("[\"tree structure\", {}{}]", nlohmann::json(repository).dump(), absent ? ", {\"absent\": true}" : ""); } auto TreeStructureRoot::ComputeHash() const -> std::size_t { std::size_t seed{}; hash_combine(&seed, kMarker); hash_combine(&seed, repository); hash_combine(&seed, absent); return seed; } auto PrecomputedRoot::Parse(nlohmann::json const& root) noexcept -> expected { if ((not root.is_array()) or root.empty()) { return unexpected{ fmt::format("The root is empty or has unsupported format: \"{}\"", root.dump())}; } if (root[0] == ComputedRoot::kMarker) { return ParsePrecomputed(root); } if (root[0] == TreeStructureRoot::kMarker) { return ParsePrecomputed(root); } return unexpected{ fmt::format("Unknown precomputed type of the root {}", root.dump())}; } auto PrecomputedRoot::ToString() const noexcept -> std::string { try { return std::visit([](auto const& r) { return r.ToString(); }, root_); } catch (...) { Ensures(false); } } auto PrecomputedRoot::GetReferencedRepository() const noexcept -> std::string { try { return std::visit([](auto const& r) { return r.repository; }, root_); } catch (...) { Ensures(false); } } auto PrecomputedRoot::ComputeHash(root_t const& root) -> std::size_t { return std::visit([](auto const& r) { return r.ComputeHash(); }, root); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/precomputed_root.hpp000066400000000000000000000122461516554100600310360ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_PRECOMPUTED_ROOT_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_PRECOMPUTED_ROOT_HPP #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" struct ComputedRoot final { static constexpr auto kMarker = "computed"; static constexpr std::size_t kSchemeLength = 5; std::string repository; std::string target_module; std::string target_name; nlohmann::json config; bool absent; [[nodiscard]] auto operator==(ComputedRoot const& other) const noexcept -> bool; [[nodiscard]] auto operator<(ComputedRoot const& other) const noexcept -> bool; [[nodiscard]] auto ToString() const -> std::string; [[nodiscard]] auto ComputeHash() const -> std::size_t; }; struct TreeStructureRoot final { static constexpr auto kMarker = "tree structure"; static constexpr std::size_t kSchemeLength = 2; static constexpr std::size_t kSchemePragmaLength = 3; std::string repository; bool absent; [[nodiscard]] auto operator==(TreeStructureRoot const& other) const noexcept -> bool; [[nodiscard]] auto operator<(TreeStructureRoot const& other) const noexcept -> bool; [[nodiscard]] auto ToString() const -> std::string; [[nodiscard]] auto ComputeHash() const -> std::size_t; }; namespace std { template struct hash; } /// \brief Generalized representation of roots that must be evaluated before the /// real build starts. class PrecomputedRoot final { public: using root_t = std::variant; explicit PrecomputedRoot() : PrecomputedRoot(ComputedRoot{}) {} explicit PrecomputedRoot(root_t root) : root_{std::move(root)}, hash_{ComputeHash(root_)} {} [[nodiscard]] static auto Parse(nlohmann::json const& root) noexcept -> expected; [[nodiscard]] static auto IsPrecomputedMarker( std::string const& marker) noexcept -> bool { return marker == ComputedRoot::kMarker or marker == TreeStructureRoot::kMarker; } [[nodiscard]] auto operator==(PrecomputedRoot const& other) const noexcept -> bool { try { return root_ == other.root_; } catch (std::exception const& e) { try { Logger::Log( LogLevel::Error, "Unexpected exception: {}", e.what()); std::terminate(); } catch (...) { std::terminate(); } } catch (...) { std::terminate(); } } [[nodiscard]] auto operator<(PrecomputedRoot const& other) const noexcept -> bool { try { return root_ < other.root_; } catch (std::exception const& e) { try { Logger::Log( LogLevel::Error, "Unexpected exception: {}", e.what()); std::terminate(); } catch (...) { std::terminate(); } } catch (...) { std::terminate(); } } [[nodiscard]] auto ToString() const noexcept -> std::string; [[nodiscard]] auto GetReferencedRepository() const noexcept -> std::string; [[nodiscard]] auto IsComputed() const noexcept -> bool { return std::holds_alternative(root_); } [[nodiscard]] auto AsComputed() const -> std::optional { if (auto const* computed = std::get_if(&root_)) { return *computed; } return std::nullopt; } [[nodiscard]] auto IsTreeStructure() const noexcept -> bool { return std::holds_alternative(root_); } [[nodiscard]] auto AsTreeStructure() const noexcept -> std::optional { if (auto const* root = std::get_if(&root_)) { return *root; } return std::nullopt; } private: root_t root_; std::size_t hash_; [[nodiscard]] static auto ComputeHash(root_t const& root) -> std::size_t; friend struct std::hash; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(PrecomputedRoot const& root) const -> std::size_t { return root.hash_; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_PRECOMPUTED_ROOT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/000077500000000000000000000000001516554100600265775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/TARGETS000066400000000000000000000021531516554100600276340ustar00rootroot00000000000000{ "resolve_symlinks_map": { "type": ["@", "rules", "CC", "library"] , "name": ["resolve_symlinks_map"] , "hdrs": ["resolve_symlinks_map.hpp"] , "srcs": ["resolve_symlinks_map.cpp"] , "deps": [ "pragma_special" , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "buildtool", "file_system", "symlinks"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "hex_string"] ] } , "pragma_special": { "type": ["@", "rules", "CC", "library"] , "name": ["pragma_special"] , "hdrs": ["pragma_special.hpp"] , "stage": ["src", "buildtool", "file_system", "symlinks"] } , "resolve_special": { "type": ["@", "rules", "CC", "library"] , "name": ["resolve_special"] , "hdrs": ["resolve_special.hpp"] , "stage": ["src", "buildtool", "file_system", "symlinks"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/pragma_special.hpp000066400000000000000000000033501516554100600322600ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_PRAGMA_SPECIAL_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_PRAGMA_SPECIAL_HPP #include #include #include #include #include /* Enum used by the resolve_symlinks_map */ /// \brief Pragma "special" value enum enum class PragmaSpecial : std::uint8_t { Ignore, ResolvePartially, ResolveCompletely }; /// \brief Pragma "special" value map, from string to enum inline std::unordered_map const kPragmaSpecialMap = {{"ignore", PragmaSpecial::Ignore}, {"resolve-partially", PragmaSpecial::ResolvePartially}, {"resolve-completely", PragmaSpecial::ResolveCompletely}}; /// \brief Pragma "special" value inverse map, from enum to string inline std::unordered_map const kPragmaSpecialInverseMap = { {PragmaSpecial::Ignore, "ignore"}, {PragmaSpecial::ResolvePartially, "resolve-partially"}, {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_PRAGMA_SPECIAL_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/resolve_special.hpp000066400000000000000000000033041516554100600324670ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_FS_RESOLVE_SPECIAL_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_FS_RESOLVE_SPECIAL_HPP #include #include #include #include #include /* Enum used by add-to-cas subcommand */ /// \brief Resolve special option value enum enum class ResolveSpecial : std::uint8_t { // Ignore special entries Ignore, // Allow symlinks confined to tree and resolve the ones that are upwards TreeUpwards, // Allow symlinks confined to tree and resolve them TreeAll, // Unconditionally allow symlinks and resolve them All }; /// \brief ResolveSpecial value map, from string to enum inline std::unordered_map const kResolveSpecialMap = {{"ignore", ResolveSpecial::Ignore}, {"tree-upwards", ResolveSpecial::TreeUpwards}, {"tree-all", ResolveSpecial::TreeAll}, {"all", ResolveSpecial::All}}; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_FS_RESOLVE_SPECIAL_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/resolve_symlinks_map.cpp000066400000000000000000000441631516554100600335600ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include #include #include #include "fmt/core.h" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { /// \brief Ensures that a given blob is in the target repo. /// On errors, calls logger with fatal and returns false. [[nodiscard]] auto EnsureBlobExists(GitObjectToResolve const& obj, GitRepo::TreeEntryInfo const& entry_info, ResolveSymlinksMap::LoggerPtr const& logger) -> bool { ExpectsAudit(IsBlobObject(entry_info.type)); // check if entry is in target repo auto target_git_repo = GitRepo::Open(obj.target_cas); if (not target_git_repo) { (*logger)("ResolveSymlinks: could not open target Git repository!", /*fatal=*/true); return false; } auto wrapped_logger = std::make_shared( [logger, id = entry_info.id](auto const& msg, bool fatal) { (*logger)(fmt::format("ResolveSymlinks: while checking blob {} " "exists in target Git repository:\n{}", id, msg), fatal); }); auto has_blob = target_git_repo->CheckBlobExists(entry_info.id, wrapped_logger); if (not has_blob) { return false; } if (not *has_blob) { // copy blob from source repo to target repo, if source is not target if (obj.source_cas.get() == obj.target_cas.get()) { (*logger)( fmt::format("ResolveSymlinks: unexpectedly missing blob {} in " "both source and target Git repositories", entry_info.id), /*fatal=*/true); return false; } auto source_git_repo = GitRepo::Open(obj.source_cas); if (not source_git_repo) { (*logger)("ResolveSymlinks: could not open source Git repository", /*fatal=*/true); return false; } wrapped_logger = std::make_shared( [logger, id = entry_info.id](auto const& msg, bool fatal) { (*logger)(fmt::format("ResolveSymlinks: while checking blob {} " "exists in source Git repository:\n{}", id, msg), fatal); }); auto res = source_git_repo->TryReadBlob(entry_info.id, wrapped_logger); if (not res.first) { return false; // fatal failure } if (not res.second.has_value()) { (*logger)(fmt::format("ResolveSymlinks: unexpectedly missing " "blob {} in source Git repository", entry_info.id), /*fatal=*/true); return false; } // write blob in target repository wrapped_logger = std::make_shared( [logger, id = entry_info.id](auto const& msg, bool fatal) { (*logger)(fmt::format("ResolveSymlinks: while writing blob " "{} into Git cache:\n{}", id, msg), fatal); }); if (not target_git_repo->WriteBlob(res.second.value(), wrapped_logger)) { return false; } } return true; // success! } /// \brief Method to handle entries by their known type. /// Guarantees to either call logger with fatal or call setter on returning. void ResolveKnownEntry(GitObjectToResolve const& obj, GitRepo::TreeEntryInfo const& entry_info, ResolveSymlinksMap::SetterPtr const& setter, ResolveSymlinksMap::LoggerPtr const& logger, ResolveSymlinksMap::SubCallerPtr const& subcaller) { // differentiated treatment based on object type if (IsFileObject(entry_info.type)) { // ensure target repository has the entry if (not EnsureBlobExists(obj, entry_info, logger)) { return; } // files are already resolved, so return the hash directly (*setter)(ResolvedGitObject{.id = entry_info.id, .type = entry_info.type, .path = obj.rel_path}); } else if (IsTreeObject(entry_info.type)) { // for tree types we resolve by rebuilding the tree from the // resolved children auto source_git_repo = GitRepo::Open(obj.source_cas); if (not source_git_repo) { (*logger)("ResolveSymlinks: could not open source Git repository!", /*fatal=*/true); return; } auto children = source_git_repo->ReadTree( entry_info.id, [](auto const& /*unused*/) { return true; }, /*is_hex_id=*/true, /*ignore_special=*/obj.pragma_special == PragmaSpecial::Ignore); if (not children) { (*logger)(fmt::format("ResolveSymlinks: failed to read entries of " "subtree {} in root tree {}", entry_info.id, obj.root_tree_id), /*fatal=*/true); return; } // resolve children std::vector children_info{}; children_info.reserve(children->size()); for (auto const& [raw_id, ev] : *children) { for (auto const& e : ev) { // must enforce ignore special at the tree level! if (IsNonSpecialObject(e.type) or obj.pragma_special != PragmaSpecial::Ignore) { // children info is known, so pass this forward if (IsSymlinkObject(e.type)) { if (auto target = obj.source_cas->ReadObject( raw_id, /*is_hex_id=*/false)) { children_info.emplace_back( obj.root_tree_id, obj.rel_path / e.name, obj.pragma_special, std::make_optional(GitRepo::TreeEntryInfo{ .id = ToHexString(raw_id), .type = e.type, .symlink_content = std::move(target)}), obj.source_cas, obj.target_cas); } else { (*logger)( fmt::format("ResolveSymlinks: could not read " "symlink {} in root tree {}", (obj.rel_path / e.name).string(), obj.root_tree_id), /*fatal=*/true); return; } } else { children_info.emplace_back( obj.root_tree_id, obj.rel_path / e.name, obj.pragma_special, GitRepo::TreeEntryInfo{ .id = ToHexString(raw_id), .type = e.type, .symlink_content = std::nullopt}, obj.source_cas, obj.target_cas); } } } } (*subcaller)( children_info, [children_info, parent = obj, setter, logger]( auto const& resolved_entries) { // create the entries map of the children GitRepo::tree_entries_t entries{}; auto num = resolved_entries.size(); entries.reserve(num); for (std::size_t i = 0; i < num; ++i) { auto const& p = children_info[i].rel_path; entries[*FromHexString(resolved_entries[i]->id)] .emplace_back( p.filename().string(), // we only need the name resolved_entries[i]->type); } // create the tree inside target repo, which should already be // existing. This operation is guarded internally, so no need // for extra locking auto target_git_repo = GitRepo::Open(parent.target_cas); if (not target_git_repo) { (*logger)( "ResolveSymlinks: could not open target Git " "repository!", /*fatal=*/true); return; } auto tree_raw_id = target_git_repo->CreateTree(entries); if (not tree_raw_id) { (*logger)(fmt::format("ResolveSymlinks: failed to create " "resolved tree {} in root tree {}", parent.rel_path.string(), parent.root_tree_id), /*fatal=*/true); return; } // set the resolved tree hash (*setter)(ResolvedGitObject{.id = ToHexString(*tree_raw_id), .type = ObjectType::Tree, .path = parent.rel_path}); }, logger); } else { // sanity check: cannot resolve a symlink called with ignore // special, as that can only be handled by the parent tree if (obj.pragma_special == PragmaSpecial::Ignore) { (*logger)(fmt::format("ResolveSymlinks: asked to ignore symlink {} " "in root tree {}", obj.rel_path.string(), obj.root_tree_id), /*fatal=*/true); return; } // target should have already been read if (not entry_info.symlink_content) { (*logger)(fmt::format("ResolveSymlinks: missing target of symlink " "{} in root tree {}", obj.rel_path.string(), obj.root_tree_id), /*fatal=*/true); return; } // check if link target (unresolved) is confined to the tree if (not PathIsConfined(*entry_info.symlink_content, obj.rel_path)) { (*logger)(fmt::format("ResolveSymlinks: symlink {} is not confined " "to tree {}", obj.rel_path.string(), obj.root_tree_id), /*fatal=*/true); return; } // if resolving partially, return a non-upwards symlink as-is if (obj.pragma_special == PragmaSpecial::ResolvePartially and PathIsNonUpwards(*entry_info.symlink_content)) { // ensure target repository has the entry if (not EnsureBlobExists(obj, entry_info, logger)) { return; } // return as symlink object (*setter)(ResolvedGitObject{.id = entry_info.id, .type = ObjectType::Symlink, .path = obj.rel_path}); return; } // resolve the target auto n_target = ToNormalPath(obj.rel_path.parent_path() / *entry_info.symlink_content); (*subcaller)( {GitObjectToResolve(obj.root_tree_id, n_target, obj.pragma_special, /*known_info=*/std::nullopt, obj.source_cas, obj.target_cas)}, [setter](auto const& values) { (*setter)(ResolvedGitObject{*values[0]}); }, logger); } } } // namespace auto CreateResolveSymlinksMap() -> ResolveSymlinksMap { auto resolve_symlinks = [](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { auto entry_info = key.known_info; if (not entry_info) { // look up entry by its relative path inside root tree if not known auto source_git_repo = GitRepo::Open(key.source_cas); if (not source_git_repo) { (*logger)( "ResolveSymlinks: could not open source Git repository!", /*fatal=*/true); return; } entry_info = source_git_repo->GetObjectByPathFromTree( key.root_tree_id, key.rel_path); } // differentiate between existing path and non-existing if (entry_info) { ResolveKnownEntry(key, *entry_info, setter, logger, subcaller); } else { // non-existing paths come from symlinks, so treat accordingly; // sanity check: pragma ignore special should not be set if here if (key.pragma_special == PragmaSpecial::Ignore) { (*logger)( fmt::format("ResolveSymlinks: asked to ignore indirect " "symlink path {} in root tree {}", key.rel_path.string(), key.root_tree_id), /*fatal=*/true); return; } auto parent_path = key.rel_path.parent_path(); if (parent_path == key.rel_path) { (*logger)(fmt::format("ResolveSymlinks: found unresolved path " "{} in root tree {}", key.rel_path.string(), key.root_tree_id), /*fatal=*/true); return; } // resolve parent (*subcaller)( {GitObjectToResolve(key.root_tree_id, parent_path, key.pragma_special, /*known_info=*/std::nullopt, key.source_cas, key.target_cas)}, [key, parent_path, filename = key.rel_path.filename(), setter, logger, subcaller](auto const& values) { auto resolved_parent = *values[0]; // parent must be a tree if (not IsTreeObject(resolved_parent.type)) { (*logger)( fmt::format("ResolveSymlinks: path {} in root tree " "{} failed to resolve to a tree", parent_path.string(), key.root_tree_id), /*fatal=*/true); return; } // check if filename exists in resolved parent tree auto target_git_repo = GitRepo::Open(key.target_cas); if (not target_git_repo) { (*logger)( "ResolveSymlinks: could not open Git cache " "repository!", /*fatal=*/true); return; } auto entry_info = target_git_repo->GetObjectByPathFromTree( resolved_parent.id, filename); if (entry_info) { ResolveKnownEntry( GitObjectToResolve(key.root_tree_id, resolved_parent.path / filename, key.pragma_special, /*known_info=*/std::nullopt, key.source_cas, key.target_cas), std::move(*entry_info), setter, logger, subcaller); } else { // report unresolvable (*logger)( fmt::format( "ResolveSymlinks: reached unresolvable " "path {} in root tree {}", (resolved_parent.path / filename).string(), key.root_tree_id), /*fatal=*/true); } }, logger); } }; return AsyncMapConsumer( resolve_symlinks); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp000066400000000000000000000116071516554100600335620ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_RESOLVE_SYMLINKS_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_RESOLVE_SYMLINKS_MAP_HPP #include #include #include #include #include #include // std::move #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/path_hash.hpp" /// \brief Information needed to resolve an object (blob or tree) given its /// path relative to the path of a root tree in a given CAS. The unresolved /// entries should be available in the specified source Git repository, and the /// resolved entries being made available in the target Git repository. struct GitObjectToResolve { // hash of the root tree std::string root_tree_id; /* key */ // path of this object relative to root tree, in normal form std::filesystem::path rel_path{"."}; /* key */ // how the tree should be resolved PragmaSpecial pragma_special{}; /* key */ // sometimes the info of the object at the required path is already known, // so leverage this to avoid extra work std::optional known_info{std::nullopt}; // object db to use as source of unresolved entries; it is guaranteed that // this repository is treated as read-only if it differs from target_cas GitCASPtr source_cas; // object db to use as target for resolved entries; can be the same as // source_cas and usually it is the Git cache; as the caller has access to // such a pointer, it reduces the overhead from opening the Git cache often GitCASPtr target_cas; GitObjectToResolve() = default; // needed for cycle detection only! GitObjectToResolve(std::string root_tree_id_, std::filesystem::path const& rel_path_, PragmaSpecial const& pragma_special_, std::optional known_info_, GitCASPtr source_cas_, GitCASPtr target_cas_) : root_tree_id{std::move(root_tree_id_)}, rel_path{ToNormalPath(rel_path_)}, pragma_special{pragma_special_}, known_info{std::move(known_info_)}, source_cas{std::move(source_cas_)}, target_cas{std::move(target_cas_)} {}; [[nodiscard]] auto operator==( GitObjectToResolve const& other) const noexcept -> bool { return root_tree_id == other.root_tree_id and rel_path == other.rel_path and pragma_special == other.pragma_special; } }; /// \brief For a possibly initially unresolved path by the end we should be able /// to know its hash, its type, and its now resolved path. struct ResolvedGitObject { std::string id; ObjectType type; std::filesystem::path path; }; /// \brief Maps information about a Git object to its Git ID, type, and path as /// part of a Git tree where symlinks have been resolved according to the given /// pragma value. /// Returns a nullopt only if called on a symlink with pragma ignore special. /// \note Call the map with type Tree and path "." to resolve a Git tree. using ResolveSymlinksMap = AsyncMapConsumer; // use explicit cast to std::function to allow template deduction when used static const std::function kGitObjectToResolvePrinter = [](GitObjectToResolve const& x) -> std::string { return x.rel_path; }; [[nodiscard]] auto CreateResolveSymlinksMap() -> ResolveSymlinksMap; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const GitObjectToResolve& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.root_tree_id); hash_combine(&seed, ct.rel_path); hash_combine(&seed, ct.pragma_special); return seed; } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_RESOLVE_SYMLINKS_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/graph_traverser/000077500000000000000000000000001516554100600256015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/graph_traverser/TARGETS000066400000000000000000000031741516554100600266420ustar00rootroot00000000000000{ "graph_traverser": { "type": ["@", "rules", "CC", "library"] , "name": ["graph_traverser"] , "hdrs": ["graph_traverser.hpp"] , "srcs": ["graph_traverser.cpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "tree"] , ["src/buildtool/common", "tree_overlay"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/execution_engine/executor", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] ] , "stage": ["src", "buildtool", "graph_traverser"] , "private-deps": [ ["src/buildtool/common", "artifact_blob"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/utils", "subobject"] , ["src/buildtool/execution_engine/executor", "executor"] , ["src/buildtool/execution_engine/traverser", "traverser"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "jsonfs"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "json"] , ["src/utils/cpp", "path"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/graph_traverser/graph_traverser.cpp000066400000000000000000000704131516554100600315100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/graph_traverser/graph_traverser.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/utils/subobject.hpp" #include "src/buildtool/execution_engine/executor/executor.hpp" #include "src/buildtool/execution_engine/traverser/traverser.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/jsonfs.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/json.hpp" #include "src/utils/cpp/path.hpp" auto GraphTraverser::BuildAndStage( std::map const& artifact_descriptions, std::map const& runfile_descriptions, std::vector&& action_descriptions, std::vector&& blobs, std::vector&& trees, std::vector&& tree_overlays, std::vector&& extra_artifacts) const -> std::optional { DependencyGraph graph; // must outlive artifact_nodes auto artifacts = BuildArtifacts(&graph, artifact_descriptions, runfile_descriptions, std::move(action_descriptions), std::move(trees), std::move(tree_overlays), std::move(blobs), extra_artifacts); if (not artifacts) { return std::nullopt; } auto [rel_paths, artifact_nodes, extra_nodes] = *artifacts; auto const object_infos = CollectObjectInfos(artifact_nodes, logger_); auto extra_infos = CollectObjectInfos(extra_nodes, logger_); if (not object_infos or not extra_infos) { return std::nullopt; } Expects(extra_artifacts.size() == extra_infos->size()); std::unordered_map infos; infos.reserve(extra_infos->size()); std::transform( std::make_move_iterator(extra_artifacts.begin()), std::make_move_iterator(extra_artifacts.end()), std::make_move_iterator(extra_infos->begin()), std::inserter(infos, infos.end()), std::make_pair); bool failed_artifacts = std::any_of( object_infos->begin(), object_infos->end(), [](auto const& info) { return info.failed; }); if (not clargs_.stage) { PrintOutputs("Artifacts built, logical paths are:", rel_paths, artifact_nodes, runfile_descriptions); MaybePrintToStdout( rel_paths, artifact_nodes, artifact_descriptions.size() == 1 ? std::optional{artifact_descriptions.begin() ->first} : std::nullopt); return BuildResult{.output_paths = std::move(rel_paths), .extra_infos = std::move(infos), .failed_artifacts = failed_artifacts}; } if (clargs_.stage->remember) { if (not context_.apis->remote->ParallelRetrieveToCas( *object_infos, *context_.apis->local, clargs_.jobs, true)) { Logger::Log( logger_, LogLevel::Warning, "Failed to copy objects to CAS"); } } auto output_paths = RetrieveOutputs(rel_paths, *object_infos); if (not output_paths) { return std::nullopt; } PrintOutputs("Artifacts can be found in:", *output_paths, artifact_nodes, runfile_descriptions); MaybePrintToStdout( rel_paths, artifact_nodes, artifact_descriptions.size() == 1 ? std::optional{artifact_descriptions.begin()->first} : std::nullopt); return BuildResult{.output_paths = *output_paths, .extra_infos = std::move(infos), .failed_artifacts = failed_artifacts}; } auto GraphTraverser::BuildAndStage( std::filesystem::path const& graph_description, nlohmann::json const& artifacts) const -> std::optional { // Read blobs to upload and actions from graph description file auto desc = ReadGraphDescription(graph_description, logger_); if (not desc) { return std::nullopt; } auto [blobs, tree_descs, actions, tree_overlay_descs] = *std::move(desc); HashFunction::Type const hash_type = context_.apis->local->GetHashType(); std::vector action_descriptions{}; action_descriptions.reserve(actions.size()); for (auto const& [id, description] : actions.items()) { auto action = ActionDescription::FromJson(hash_type, id, description); if (not action) { return std::nullopt; // Error already logged } action_descriptions.emplace_back(std::move(*action)); } std::vector trees{}; for (auto const& [id, description] : tree_descs.items()) { auto tree = Tree::FromJson(hash_type, id, description); if (not tree) { return std::nullopt; } trees.emplace_back(std::move(*tree)); } std::vector tree_overlays{}; for (auto const& [id, description] : tree_overlay_descs.items()) { auto tree_overlay = TreeOverlay::FromJson(hash_type, id, description); if (not tree_overlay) { return std::nullopt; } tree_overlays.emplace_back(std::move(*tree_overlay)); } std::map artifact_descriptions{}; for (auto const& [rel_path, description] : artifacts.items()) { auto artifact = ArtifactDescription::FromJson(hash_type, description); if (not artifact) { return std::nullopt; // Error already logged } artifact_descriptions.emplace(rel_path, std::move(*artifact)); } return BuildAndStage(artifact_descriptions, {}, std::move(action_descriptions), std::move(blobs), std::move(trees), std::move(tree_overlays)); } auto GraphTraverser::ReadGraphDescription( std::filesystem::path const& graph_description, Logger const* logger) -> std::optional> { auto const graph_description_opt = Json::ReadFile(graph_description); if (not graph_description_opt.has_value()) { Logger::Log(logger, LogLevel::Error, "parsing graph from {}", graph_description.string()); return std::nullopt; } auto blobs_opt = ExtractValueAs>( *graph_description_opt, "blobs", [logger](std::string const& s) { Logger::Log(logger, LogLevel::Error, "{}\ncan not retrieve value for \"blobs\" from " "graph description.", s); }); auto trees_opt = ExtractValueAs( *graph_description_opt, "trees", [logger](std::string const& s) { Logger::Log(logger, LogLevel::Error, "{}\ncan not retrieve value for \"trees\" from " "graph description.", s); }); auto actions_opt = ExtractValueAs( *graph_description_opt, "actions", [logger](std::string const& s) { Logger::Log(logger, LogLevel::Error, "{}\ncan not retrieve value for \"actions\" from " "graph description.", s); }); std::optional tree_overlays_opt; if (graph_description_opt->contains("tree_overlays")) { tree_overlays_opt = ExtractValueAs( *graph_description_opt, "tree_overlays", [logger](std::string const& s) { Logger::Log( logger, LogLevel::Error, "{}\ncan not retrieve value for \"tree_overlays\" from " "graph description.", s); }); } else { tree_overlays_opt = nlohmann::json::object(); } if (not blobs_opt or not trees_opt or not actions_opt or not tree_overlays_opt) { return std::nullopt; } return std::make_tuple(std::move(*blobs_opt), std::move(*trees_opt), std::move(*actions_opt), std::move(*tree_overlays_opt)); } auto GraphTraverser::UploadBlobs( std::vector&& blobs) const noexcept -> bool { std::unordered_set container; HashFunction const hash_function{context_.apis->remote->GetHashType()}; for (auto& content : blobs) { auto blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, std::move(content)); if (not blob.has_value()) { logger_->Emit(LogLevel::Trace, "Failed to create ArtifactBlob"); return false; } Logger::Log(logger_, LogLevel::Trace, [&]() { return fmt::format( "Will upload blob, its digest has id {} and size {}.", blob->GetDigest().hash(), blob->GetDigest().size()); }); // Store and/or upload blob, taking into account the maximum // transfer size. if (not UpdateContainerAndUpload( &container, *std::move(blob), /*exception_is_fatal=*/true, [&api = context_.apis->remote]( std::unordered_set&& blobs) { return api->Upload(std::move(blobs)); }, logger_)) { return false; } } // Upload remaining blobs. auto result = context_.apis->remote->Upload(std::move(container)); Logger::Log(logger_, LogLevel::Trace, [&]() { std::stringstream msg{}; msg << (result ? "Finished" : "Failed") << " upload of\n"; for (auto const& blob : blobs) { msg << " - " << nlohmann::json(blob).dump() << "\n"; } return msg.str(); }); return result; } auto GraphTraverser::AddArtifactsToRetrieve( gsl::not_null const& g, std::map const& artifacts, std::map const& runfiles) -> std::optional, std::vector>> { std::vector rel_paths; std::vector ids; auto total_size = artifacts.size() + runfiles.size(); rel_paths.reserve(total_size); ids.reserve(total_size); auto add_and_get_info = [&g, &rel_paths, &ids]( std::map const& descriptions) -> bool { for (auto const& [rel_path, artifact] : descriptions) { rel_paths.emplace_back(rel_path); ids.emplace_back(g->AddArtifact(artifact)); } return true; }; if (add_and_get_info(artifacts) and add_and_get_info(runfiles)) { return std::make_pair(std::move(rel_paths), std::move(ids)); } return std::nullopt; } auto GraphTraverser::Traverse( DependencyGraph const& g, std::vector const& artifact_ids) const -> bool { Executor executor{&context_, logger_, clargs_.build.timeout}; bool traversing{}; std::atomic done = false; std::atomic failed = false; std::condition_variable cv{}; auto observer = std::thread([this, &done, &cv]() { reporter_(&done, &cv); }); { Traverser t{executor, g, clargs_.jobs, &failed}; traversing = t.Traverse({std::begin(artifact_ids), std::end(artifact_ids)}); } done = true; cv.notify_all(); observer.join(); return traversing and not failed; } auto GraphTraverser::TraverseRebuild( DependencyGraph const& g, std::vector const& artifact_ids) const -> bool { Rebuilder executor{&context_, clargs_.build.timeout}; bool traversing{false}; std::atomic done = false; std::atomic failed = false; std::condition_variable cv{}; auto observer = std::thread([this, &done, &cv]() { reporter_(&done, &cv); }); { Traverser t{executor, g, clargs_.jobs, &failed}; traversing = t.Traverse({std::begin(artifact_ids), std::end(artifact_ids)}); } done = true; cv.notify_all(); observer.join(); if (traversing and not failed and clargs_.rebuild->dump_flaky) { std::ofstream file{*clargs_.rebuild->dump_flaky}; file << executor.DumpFlakyActions().dump(2); } return traversing and not failed; } auto GraphTraverser::GetArtifactNodes( DependencyGraph const& g, std::vector const& artifact_ids, Logger const* logger) noexcept -> std::optional> { std::vector nodes{}; for (auto const& art_id : artifact_ids) { auto const* node = g.ArtifactNodeWithId(art_id); if (node == nullptr) { Logger::Log(logger, LogLevel::Error, "Artifact {} not found in graph.", art_id); return std::nullopt; } nodes.push_back(node); } return nodes; } void GraphTraverser::LogStatistics() const noexcept { auto& stats = *context_.statistics; if (clargs_.rebuild) { std::stringstream ss{}; ss << stats.RebuiltActionComparedCounter() << " actions compared with cache"; if (stats.ActionsFlakyCounter() > 0) { ss << ", " << stats.ActionsFlakyCounter() << " flaky actions found"; ss << " (" << stats.ActionsFlakyTaintedCounter() << " of which tainted)"; } if (stats.RebuiltActionMissingCounter() > 0) { ss << ", no cache entry found for " << stats.RebuiltActionMissingCounter() << " actions"; } ss << "."; Logger::Log(logger_, LogLevel::Info, ss.str()); } else { Logger::Log(logger_, LogLevel::Info, "Processed {} actions, {} cache hits.", stats.ActionsQueuedCounter(), stats.ActionsCachedCounter()); } } auto GraphTraverser::BuildArtifacts( gsl::not_null const& graph, std::map const& artifacts, std::map const& runfiles, std::vector&& actions, std::vector&& trees, std::vector&& tree_overlays, std::vector&& blobs, std::vector const& extra_artifacts) const -> std::optional< std::tuple, std::vector, std::vector>> { if (not UploadBlobs(std::move(blobs))) { return std::nullopt; } auto artifact_infos = AddArtifactsToRetrieve(graph, artifacts, runfiles); if (not artifact_infos) { return std::nullopt; } auto& [output_paths, artifact_ids] = *artifact_infos; // Add extra artifacts to ids to build artifact_ids.reserve(artifact_ids.size() + extra_artifacts.size()); for (auto const& artifact : extra_artifacts) { artifact_ids.emplace_back(graph->AddArtifact(artifact)); } std::vector tree_actions{}; tree_actions.reserve(trees.size() + tree_overlays.size()); for (auto const& tree : trees) { tree_actions.emplace_back(tree->Action()); } for (auto const& tree : tree_overlays) { tree_actions.emplace_back(tree->Action()); } if (not graph->Add(actions) or not graph->Add(tree_actions)) { Logger::Log(logger_, LogLevel::Error, [&actions]() { auto json = nlohmann::json::array(); for (auto const& desc : actions) { json.push_back(desc->ToJson()); } return fmt::format( "could not build the dependency graph from the actions " "described in {}.", json.dump()); }); return std::nullopt; } if (clargs_.rebuild ? not TraverseRebuild(*graph, artifact_ids) : not Traverse(*graph, artifact_ids)) { Logger::Log(logger_, LogLevel::Error, "Build failed."); return std::nullopt; } LogStatistics(); auto artifact_nodes = GetArtifactNodes(*graph, artifact_ids, logger_); if (not artifact_nodes) { return std::nullopt; } // split extra artifacts' nodes from artifact nodes auto const it_extra = std::next(artifact_nodes->begin(), static_cast(output_paths.size())); auto extra_nodes = std::vector{ std::make_move_iterator(it_extra), std::make_move_iterator(artifact_nodes->end())}; artifact_nodes->erase(it_extra, artifact_nodes->end()); return std::make_tuple(std::move(output_paths), std::move(*artifact_nodes), std::move(extra_nodes)); } auto GraphTraverser::PrepareOutputPaths( std::vector const& rel_paths) const -> std::optional> { std::vector output_paths{}; output_paths.reserve(rel_paths.size()); for (auto const& rel_path : rel_paths) { output_paths.emplace_back(clargs_.stage->output_dir / rel_path); } return output_paths; } auto GraphTraverser::CollectObjectInfos( std::vector const& artifact_nodes, Logger const* logger) -> std::optional> { std::vector object_infos; object_infos.reserve(artifact_nodes.size()); for (auto const* art_ptr : artifact_nodes) { auto const& info = art_ptr->Content().Info(); if (info) { object_infos.push_back(*info); } else { Logger::Log(logger, LogLevel::Error, "artifact {} could not be retrieved, it can not be " "found in CAS.", art_ptr->Content().Id()); return std::nullopt; } } return object_infos; } auto GraphTraverser::RetrieveOutputs( std::vector const& rel_paths, std::vector const& object_infos) const -> std::optional> { // Create output directory if (not FileSystemManager::CreateDirectory(clargs_.stage->output_dir)) { return std::nullopt; // Message logged in the file system manager } auto output_paths = PrepareOutputPaths(rel_paths); if (not output_paths or not context_.apis->remote->RetrieveToPaths( object_infos, *output_paths, &*context_.apis->local)) { Logger::Log(logger_, LogLevel::Error, "Could not retrieve outputs."); return std::nullopt; } return output_paths; } void GraphTraverser::PrintOutputs( std::string message, std::vector const& paths, std::vector const& artifact_nodes, std::map const& runfiles) const { std::string msg_failed{"Failed artifacts:"}; bool failed{false}; nlohmann::json json{}; for (std::size_t pos = 0; pos < paths.size(); ++pos) { auto path = paths[pos].string(); auto id = IdentifierToString(artifact_nodes[pos]->Content().Id()); if (clargs_.build.show_runfiles or not runfiles.contains(clargs_.stage ? std::filesystem::proximate( path, clargs_.stage->output_dir) .string() : path)) { auto info = artifact_nodes[pos]->Content().Info(); if (info) { message += fmt::format("\n {} {}", path, info->ToString()); if (info->failed) { msg_failed += fmt::format("\n {} {}", path, info->ToString()); failed = true; } if (not clargs_.build.dump_artifacts.empty()) { json[path] = info->ToJson(); } } else { Logger::Log(logger_, LogLevel::Error, "Missing info for artifact {}.", id); } } } if (not clargs_.build.show_runfiles and not runfiles.empty()) { message += fmt::format("\n({} runfiles omitted.)", runfiles.size()); } Logger::Log(logger_, LogLevel::Info, "{}", message); if (failed) { Logger::Log(logger_, LogLevel::Info, "{}", msg_failed); } for (auto const& location : clargs_.build.dump_artifacts) { if (location == "-") { std::cout << std::setw(2) << json << std::endl; } else { std::ofstream os(location); os << std::setw(2) << json << std::endl; } } } void GraphTraverser::MaybePrintToStdout( std::vector const& paths, std::vector const& artifacts, std::optional const& unique_artifact) const { if (clargs_.build.print_to_stdout) { auto const& remote = *context_.apis->remote; for (std::size_t i = 0; i < paths.size(); i++) { if (paths[i] == *(clargs_.build.print_to_stdout)) { auto info = artifacts[i]->Content().Info(); if (info) { if (not remote.RetrieveToFds({*info}, {dup(fileno(stdout))}, /*raw_tree=*/false, &*context_.apis->local)) { Logger::Log(logger_, LogLevel::Error, "Failed to retrieve {}", *(clargs_.build.print_to_stdout)); } } else { Logger::Log(logger_, LogLevel::Error, "Failed to obtain object information for {}", *(clargs_.build.print_to_stdout)); } return; } } // Not directly an artifact, hence check if the path is contained in // some artifact auto target_path = ToNormalPath(std::filesystem::path{*clargs_.build.print_to_stdout}) .relative_path(); for (std::size_t i = 0; i < paths.size(); i++) { auto const& path = paths[i]; auto relpath = target_path.lexically_relative(path); if ((not relpath.empty()) and *relpath.begin() != "..") { Logger::Log(logger_, LogLevel::Info, "'{}' not a direct logical path of the specified " "target; will take subobject '{}' of '{}'", *(clargs_.build.print_to_stdout), relpath.string(), path.string()); auto info = artifacts[i]->Content().Info(); if (info) { auto new_info = RetrieveSubPathId(*info, *context_.apis, relpath); if (new_info) { if (not remote.RetrieveToFds({*new_info}, {dup(fileno(stdout))}, /*raw_tree=*/false, &*context_.apis->local)) { Logger::Log(logger_, LogLevel::Error, "Failed to retrieve artifact {} at " "path '{}' of '{}'", new_info->ToString(), relpath.string(), path.string()); } } } else { Logger::Log(logger_, LogLevel::Error, "Failed to obtain object information for {}", *(clargs_.build.print_to_stdout)); } return; } } Logger::Log(logger_, LogLevel::Warning, "{} not a logical path of the specified target", *(clargs_.build.print_to_stdout)); } else if (clargs_.build.print_unique) { if (unique_artifact) { auto const& remote = *context_.apis->remote; std::optional info = std::nullopt; for (std::size_t i = 0; i < paths.size(); i++) { if (paths[i] == unique_artifact) { info = artifacts[i]->Content().Info(); } } if (info) { if (not remote.RetrieveToFds({*info}, {dup(fileno(stdout))}, /*raw_tree=*/false, &*context_.apis->local)) { Logger::Log(logger_, LogLevel::Error, "Failed to retrieve {}", *unique_artifact); } } else { Logger::Log(logger_, LogLevel::Error, "Failed to obtain object information for {}", *unique_artifact); } return; } Logger::Log(logger_, LogLevel::Info, "Target does not have precisely one artifact."); } } #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/graph_traverser/graph_traverser.hpp000066400000000000000000000215471516554100600315210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP #define INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" class GraphTraverser { public: struct CommandLineArguments { std::size_t jobs; BuildArguments build; std::optional stage; std::optional rebuild; }; struct BuildResult { std::vector output_paths; // Object infos of extra artifacts requested to build. std::unordered_map extra_infos; bool failed_artifacts; }; explicit GraphTraverser( CommandLineArguments clargs, gsl::not_null const& context, progress_reporter_t reporter, Logger const* logger = nullptr) : clargs_{std::move(clargs)}, context_{*context}, reporter_{std::move(reporter)}, logger_{logger} {} /// \brief Parses actions and blobs into graph, traverses it and retrieves /// outputs specified by command line arguments. /// \param artifact_descriptions Artifacts to build (and stage). /// \param runfile_descriptions Runfiles to build (and stage). /// \param action_descriptions All required actions for building. /// \param blobs Blob artifacts to upload before the build. /// \param trees Tree artifacts to compute graph nodes from. /// \param extra_artifacts Extra artifacts to obtain object infos for. [[nodiscard]] auto BuildAndStage( std::map const& artifact_descriptions, std::map const& runfile_descriptions, std::vector&& action_descriptions, std::vector&& blobs, std::vector&& trees, std::vector&& tree_overlays, std::vector&& extra_artifacts = {}) const -> std::optional; /// \brief Parses graph description into graph, traverses it and retrieves /// outputs specified by command line arguments [[nodiscard]] auto BuildAndStage( std::filesystem::path const& graph_description, nlohmann::json const& artifacts) const -> std::optional; private: CommandLineArguments const clargs_; ExecutionContext const& context_; progress_reporter_t reporter_; Logger const* logger_{nullptr}; /// \brief Reads contents of graph description file as json object. In case /// the description is missing "blobs" or "actions" key/value pairs or they /// can't be retrieved with the appropriate types, execution is terminated /// after logging error /// \returns A pair containing the blobs to upload (as a vector of strings) /// and the actions as a json object. [[nodiscard]] static auto ReadGraphDescription( std::filesystem::path const& graph_description, Logger const* logger) -> std::optional>; /// \brief Requires for the executor to upload blobs to CAS. In the case any /// of the uploads fails, execution is terminated /// \param[in] blobs blobs to be uploaded [[nodiscard]] auto UploadBlobs( std::vector&& blobs) const noexcept -> bool; /// \brief Adds the artifacts to be retrieved to the graph /// \param[in] g dependency graph /// \param[in] artifacts output artifact map /// \param[in] runfiles output runfile map /// \returns pair of vectors where the first vector contains the absolute /// paths to which the artifacts will be retrieved and the second one /// contains the ids of the artifacts to be retrieved [[nodiscard]] static auto AddArtifactsToRetrieve( gsl::not_null const& g, std::map const& artifacts, std::map const& runfiles) -> std::optional, std::vector>>; /// \brief Traverses the graph. In case any of the artifact ids /// specified by the command line arguments is duplicated, execution is /// terminated. [[nodiscard]] auto Traverse( DependencyGraph const& g, std::vector const& artifact_ids) const -> bool; [[nodiscard]] auto TraverseRebuild( DependencyGraph const& g, std::vector const& artifact_ids) const -> bool; /// \brief Retrieves nodes corresponding to artifacts with ids in artifacts. /// In case any of the identifiers doesn't correspond to a node inside the /// graph, we write out error message and stop execution with failure code [[nodiscard]] static auto GetArtifactNodes( DependencyGraph const& g, std::vector const& artifact_ids, Logger const* logger) noexcept -> std::optional>; void LogStatistics() const noexcept; [[nodiscard]] auto BuildArtifacts( gsl::not_null const& graph, std::map const& artifacts, std::map const& runfiles, std::vector&& actions, std::vector&& trees, std::vector&& tree_overlays, std::vector&& blobs, std::vector const& extra_artifacts = {}) const -> std::optional< std::tuple, std::vector, std::vector>>; [[nodiscard]] auto PrepareOutputPaths( std::vector const& rel_paths) const -> std::optional>; [[nodiscard]] static auto CollectObjectInfos( std::vector const& artifact_nodes, Logger const* logger) -> std::optional>; /// \brief Asks execution API to copy output artifacts to paths specified by /// command line arguments and writes location info. In case the executor /// couldn't retrieve any of the outputs, execution is terminated. [[nodiscard]] auto RetrieveOutputs( std::vector const& rel_paths, std::vector const& object_infos) const -> std::optional>; void PrintOutputs( std::string message, std::vector const& paths, std::vector const& artifact_nodes, std::map const& runfiles) const; void MaybePrintToStdout( std::vector const& paths, std::vector const& artifacts, std::optional const& unique_artifact) const; }; #endif #endif // INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/000077500000000000000000000000001516554100600240315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/TARGETS000066400000000000000000000011761516554100600250720ustar00rootroot00000000000000{ "log_level": { "type": ["@", "rules", "CC", "library"] , "name": ["log_level"] , "hdrs": ["log_level.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "logging"] } , "logging": { "type": ["@", "rules", "CC", "library"] , "name": ["logging"] , "hdrs": [ "log_config.hpp" , "log_sink.hpp" , "log_sink_cmdline.hpp" , "log_sink_file.hpp" , "logger.hpp" ] , "deps": ["log_level", ["@", "fmt", "", "fmt"], ["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "logging"] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/log_config.hpp000066400000000000000000000054521516554100600266560ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP #include #include #include #include #include #include // std::move #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" /// \brief Global static logging configuration. /// The entire class is thread-safe. class LogConfig { struct ConfigData { std::mutex mutex; LogLevel log_limit{LogLevel::Info}; std::vector sinks; std::vector factories; }; public: /// \brief Set the log limit. static void SetLogLimit(LogLevel level) noexcept { Data().log_limit = level; } /// \brief Replace all configured sinks. /// NOTE: Reinitializes all internal factories. static void SetSinks(std::vector&& factories) noexcept { auto& data = Data(); std::lock_guard lock{data.mutex}; data.sinks.clear(); data.sinks.reserve(factories.size()); std::transform(factories.cbegin(), factories.cend(), std::back_inserter(data.sinks), [](auto& f) { return f(); }); data.factories = std::move(factories); } /// \brief Add new a new sink. static void AddSink(LogSinkFactory&& factory) noexcept { auto& data = Data(); std::lock_guard lock{data.mutex}; data.sinks.push_back(factory()); data.factories.push_back(std::move(factory)); } /// \brief Get the currently configured log limit. [[nodiscard]] static auto LogLimit() noexcept -> LogLevel { return Data().log_limit; } /// \brief Get sink instances for all configured sink factories. [[nodiscard]] static auto Sinks() noexcept -> std::vector { auto& data = Data(); std::lock_guard lock{data.mutex}; return data.sinks; } private: [[nodiscard]] static auto Data() noexcept -> ConfigData& { static ConfigData instance{}; return instance; } }; #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/log_level.hpp000066400000000000000000000054751516554100600265250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP #include #include #include #include #include #include "gsl/gsl" enum class LogLevel : std::uint8_t { Error, ///< Error messages, fatal errors Warning, ///< Warning messages, recoverable situations that shouldn't occur Info, ///< Informative messages, such as reporting status or statistics Progress, ///< Information about the current progress of the build Performance, ///< Information about performance issues Debug, ///< Debug messages, such as details from internal processes Trace ///< Trace messages, verbose details such as function calls }; constexpr auto kFirstLogLevel = LogLevel::Error; constexpr auto kLastLogLevel = LogLevel::Trace; [[nodiscard]] static inline auto ToLogLevel( std::underlying_type_t level) -> LogLevel { return std::min(std::max(static_cast(level), kFirstLogLevel), kLastLogLevel); } [[nodiscard]] static inline auto ToLogLevel(double level) -> LogLevel { if (level < static_cast(kFirstLogLevel)) { return kFirstLogLevel; } if (level > static_cast(kLastLogLevel)) { return kLastLogLevel; } // Now we're in range, so we can round and cast. try { return ToLogLevel( static_cast>(std::lround(level))); } catch (...) { // should not happen, but chose the most conservative value return kLastLogLevel; } } [[nodiscard]] static inline auto LogLevelToString(LogLevel level) -> std::string { switch (level) { case LogLevel::Error: return "ERROR"; case LogLevel::Warning: return "WARN"; case LogLevel::Info: return "INFO"; case LogLevel::Progress: return "PROG"; case LogLevel::Performance: return "PERF"; case LogLevel::Debug: return "DEBUG"; case LogLevel::Trace: return "TRACE"; } Ensures(false); // unreachable } #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/log_sink.hpp000066400000000000000000000035301516554100600263500ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP #include #include #include #include #include "src/buildtool/logging/log_level.hpp" // forward declaration class Logger; class ILogSink { public: using Ptr = std::shared_ptr; ILogSink() noexcept = default; ILogSink(ILogSink const&) = delete; ILogSink(ILogSink&&) = delete; auto operator=(ILogSink const&) -> ILogSink& = delete; auto operator=(ILogSink&&) -> ILogSink& = delete; virtual ~ILogSink() noexcept = default; /// \brief Thread-safe emitting of log messages. /// Logger might be 'nullptr' if called from the global context. virtual void Emit(Logger const* logger, LogLevel level, std::string const& msg) const noexcept = 0; protected: /// \brief Helper class for line iteration with std::istream_iterator. class Line : public std::string { friend auto operator>>(std::istream& is, Line& line) -> std::istream& { return std::getline(is, line); } }; }; using LogSinkFactory = std::function; #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/log_sink_cmdline.hpp000066400000000000000000000111651516554100600300460ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP #include #include #include #include #include #include #include #include #include "fmt/color.h" #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" #include "src/buildtool/logging/logger.hpp" class LogSinkCmdLine final : public ILogSink { public: static auto CreateFactory(bool colored = true, std::optional restrict_level = std::nullopt) -> LogSinkFactory { return [=]() { return std::make_shared(colored, restrict_level); }; } explicit LogSinkCmdLine(bool colored, std::optional restrict_level) noexcept : colored_{colored}, restrict_level_{restrict_level} {} ~LogSinkCmdLine() noexcept final = default; LogSinkCmdLine(LogSinkCmdLine const&) noexcept = delete; LogSinkCmdLine(LogSinkCmdLine&&) noexcept = delete; auto operator=(LogSinkCmdLine const&) noexcept -> LogSinkCmdLine& = delete; auto operator=(LogSinkCmdLine&&) noexcept -> LogSinkCmdLine& = delete; /// \brief Thread-safe emitting of log messages to stderr. void Emit(Logger const* logger, LogLevel level, std::string const& msg) const noexcept final { static std::mutex mutex{}; if (restrict_level_ and (static_cast(*restrict_level_) < static_cast(level))) { return; } auto prefix = LogLevelToString(level); if (logger != nullptr) { // append logger name prefix = fmt::format("{} ({})", prefix, logger->Name()); } prefix = prefix + ":"; auto cont_prefix = std::string(prefix.size(), ' '); prefix = FormatPrefix(level, prefix); bool msg_on_continuation{false}; if (logger != nullptr and msg.find('\n') != std::string::npos) { cont_prefix = " "; msg_on_continuation = true; } { std::lock_guard lock{mutex}; if (msg_on_continuation) { fmt::print(stderr, "{}\n", prefix); prefix = cont_prefix; } using it = std::istream_iterator; std::istringstream iss{msg}; for_each(it{iss}, it{}, [&](auto const& line) { fmt::print(stderr, "{} {}\n", prefix, line); prefix = cont_prefix; }); std::fflush(stderr); } } private: bool colored_{}; std::optional restrict_level_; [[nodiscard]] auto FormatPrefix(LogLevel level, std::string const& prefix) const noexcept -> std::string { fmt::text_style style{}; if (colored_) { switch (level) { case LogLevel::Error: style = fg(fmt::color::red); break; case LogLevel::Warning: style = fg(fmt::color::orange); break; case LogLevel::Info: style = fg(fmt::color::lime_green); break; case LogLevel::Progress: style = fg(fmt::color::dark_green); break; case LogLevel::Performance: style = fg(fmt::color::light_sky_blue); break; case LogLevel::Debug: style = fg(fmt::color::sky_blue); break; case LogLevel::Trace: style = fg(fmt::color::deep_sky_blue); break; } } try { return fmt::format(style, "{}", prefix); } catch (...) { return prefix; } } }; #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/log_sink_file.hpp000066400000000000000000000122031516554100600273440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/chrono.h" #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" #include "src/buildtool/logging/logger.hpp" /// \brief Thread-safe map of mutexes. template class MutexMap { public: /// \brief Create mutex for key and run callback if successfully created. /// Callback is executed while the internal map is still held exclusively. void Create(TKey const& key, std::function const& callback) { std::lock_guard lock(mutex_); if (not map_.contains(key)) { [[maybe_unused]] auto& mutex = map_[key]; callback(); } } /// \brief Get mutex for key, creates mutex if key does not exist. [[nodiscard]] auto Get(TKey const& key) noexcept -> std::mutex& { std::lock_guard lock(mutex_); return map_[key]; } private: std::mutex mutex_; std::unordered_map map_; }; class LogSinkFile final : public ILogSink { public: enum class Mode : std::uint8_t { Append, ///< Append if log file already exists. Overwrite ///< Overwrite log file with each new program instantiation. }; static auto CreateFactory(std::filesystem::path const& file_path, Mode file_mode = Mode::Append) -> LogSinkFactory { return [=] { return std::make_shared(file_path, file_mode); }; } LogSinkFile(std::filesystem::path const& file_path, Mode file_mode) : file_path_{std::filesystem::weakly_canonical(file_path).string()} { // create file mutex for canonical path FileMutexes().Create(file_path_, [&] { if (file_mode == Mode::Overwrite) { // clear file contents if (gsl::owner file = std::fopen(file_path_.c_str(), "w")) { std::fclose(file); } } }); } ~LogSinkFile() noexcept final = default; LogSinkFile(LogSinkFile const&) noexcept = delete; LogSinkFile(LogSinkFile&&) noexcept = delete; auto operator=(LogSinkFile const&) noexcept -> LogSinkFile& = delete; auto operator=(LogSinkFile&&) noexcept -> LogSinkFile& = delete; /// \brief Thread-safe emitting of log messages to file. /// Race-conditions for file writes are resolved via a separate mutexes for /// every canonical file path shared across all instances of this class. void Emit(Logger const* logger, LogLevel level, std::string const& msg) const noexcept final { #ifdef __unix__ // support nanoseconds for timestamp timespec ts{}; clock_gettime(CLOCK_REALTIME, &ts); auto timestamp = fmt::format( "{:%Y-%m-%d %H:%M:%S}.{} UTC", fmt::gmtime(ts.tv_sec), ts.tv_nsec); #else auto timestamp = fmt::format( "{:%Y-%m-%d %H:%M:%S} UTC", fmt::gmtime(std::time(nullptr)); #endif std::ostringstream id{}; id << "thread:" << std::this_thread::get_id(); auto thread = id.str(); auto prefix = fmt::format( "{}, [{}] {}", thread, timestamp, LogLevelToString(level)); if (logger != nullptr) { // append logger name prefix = fmt::format("{} ({})", prefix, logger->Name()); } prefix = fmt::format("{}:", prefix); const auto* cont_prefix = " "; { std::lock_guard lock{FileMutexes().Get(file_path_)}; if (gsl::owner file = std::fopen(file_path_.c_str(), "a")) { using it = std::istream_iterator; std::istringstream iss{msg}; for_each(it{iss}, it{}, [&](auto const& line) { fmt::print(file, "{} {}\n", prefix, line); prefix = cont_prefix; }); std::fclose(file); } } } private: std::string file_path_; [[nodiscard]] static auto FileMutexes() noexcept -> MutexMap& { static MutexMap instance{}; return instance; } }; #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/logging/logger.hpp000066400000000000000000000150511516554100600260230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP #define INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" class Logger { public: using MessageCreateFunc = std::function; /// \brief Create logger with sink instances from LogConfig::Sinks(). explicit Logger(std::string name) noexcept : name_{std::move(name)}, log_limit_{LogConfig::LogLimit()}, sinks_{LogConfig::Sinks()} {} /// \brief Create logger with new sink instances from specified factories. Logger(std::string name, std::vector const& factories) noexcept : name_{std::move(name)}, log_limit_{LogConfig::LogLimit()} { sinks_.reserve(factories.size()); std::transform(factories.cbegin(), factories.cend(), std::back_inserter(sinks_), [](auto& f) { return f(); }); } ~Logger() noexcept = default; Logger(Logger const&) noexcept = delete; Logger(Logger&&) noexcept = delete; auto operator=(Logger const&) noexcept -> Logger& = delete; auto operator=(Logger&&) noexcept -> Logger& = delete; /// \brief Get logger name. [[nodiscard]] auto Name() const& noexcept -> std::string const& { return name_; } /// \brief Get log limit. [[nodiscard]] auto LogLimit() const noexcept -> LogLevel { return log_limit_; } /// \brief Set log limit. void SetLogLimit(LogLevel level) noexcept { log_limit_ = level; } /// \brief Emit log message from string via this logger instance. template void Emit(LogLevel level, std::string const& msg, TArgs&&... args) const noexcept { if (static_cast(level) <= static_cast(log_limit_)) { FormatAndForward( this, sinks_, level, msg, std::forward(args)...); } } /// \brief Emit log message from lambda via this logger instance. void Emit(LogLevel level, MessageCreateFunc const& msg_creator) const noexcept { if (static_cast(level) <= static_cast(log_limit_)) { FormatAndForward(this, sinks_, level, msg_creator()); } } /// \brief Log message from string via LogConfig's sinks and log limit. template static void Log(LogLevel level, std::string const& msg, TArgs&&... args) noexcept { if (static_cast(level) <= static_cast(LogConfig::LogLimit())) { FormatAndForward(nullptr, LogConfig::Sinks(), level, msg, std::forward(args)...); } } /// \brief Log message from lambda via LogConfig's sinks and log limit. static void Log(LogLevel level, MessageCreateFunc const& msg_creator) noexcept { if (static_cast(level) <= static_cast(LogConfig::LogLimit())) { FormatAndForward(nullptr, LogConfig::Sinks(), level, msg_creator()); } } /// \brief Generic logging method. Provides a common interface between the /// global logger and named instances, hidden from the outside caller. /// For named instances no global configuration is used. template static void Log(Logger const* logger, LogLevel level, std::string const& msg, TArgs&&... args) noexcept { if (static_cast(level) <= static_cast(logger != nullptr ? logger->log_limit_ : LogConfig::LogLimit())) { FormatAndForward( logger, logger != nullptr ? logger->sinks_ : LogConfig::Sinks(), level, msg, std::forward(args)...); } } /// \brief Generic logging method with provided message creator. Provides a /// common interface between the global logger and named instances, hidden /// from the outside caller. /// For named instances no global configuration is used. static void Log(Logger const* logger, LogLevel level, MessageCreateFunc const& msg_creator) noexcept { if (static_cast(level) <= static_cast(logger != nullptr ? logger->log_limit_ : LogConfig::LogLimit())) { FormatAndForward( logger, logger != nullptr ? logger->sinks_ : LogConfig::Sinks(), level, msg_creator()); } } private: std::string name_; LogLevel log_limit_{}; std::vector sinks_; /// \brief Format message and forward to sinks. template static void FormatAndForward( Logger const* logger, std::vector const& sinks, LogLevel level, std::string const& msg, // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) TArgs&&... args) noexcept { if constexpr (sizeof...(TArgs) == 0) { // forward to sinks std::for_each(sinks.cbegin(), sinks.cend(), [&](auto& sink) { sink->Emit(logger, level, msg); }); } else { // format the message auto fmsg = fmt::vformat(msg, fmt::make_format_args(args...)); // recursive call without format arguments FormatAndForward(logger, sinks, level, fmsg); } } }; #endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/000077500000000000000000000000001516554100600233275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/TARGETS000066400000000000000000000344331516554100600243720ustar00rootroot00000000000000{ "just": { "type": ["@", "rules", "CC", "binary"] , "arguments_config": ["FINAL_LDFLAGS"] , "name": ["just"] , "srcs": ["main.cpp"] , "private-deps": [ "add_to_cas" , "analyse" , "analyse_context" , "build_utils" , "cli" , "common" , "constants" , "describe" , "diagnose" , "install_cas" , "retry" , "serve" , "version" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/target_map", "absent_target_map"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "clidefaults"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/common/remote", "retry_config"] , ["src/buildtool/computed_roots", "evaluate"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , [ "src/buildtool/execution_api/execution_service" , "server_implementation" ] , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/execution_engine/executor", "context"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_context"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/profile", "profile"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "progress_reporter"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/serve_api/serve_service", "serve_server_implementation"] , ["src/buildtool/storage", "backend_description"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "file_chunker"] , ["src/buildtool/storage", "garbage_collector"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "json"] ] , "stage": ["src", "buildtool", "main"] , "private-ldflags": { "type": "++" , "$1": [ ["-Wl,-z,stack-size=8388608"] , {"type": "var", "name": "FINAL_LDFLAGS", "default": []} ] } } , "retry": { "type": ["@", "rules", "CC", "library"] , "name": ["retry"] , "hdrs": ["retry.hpp"] , "srcs": ["retry.cpp"] , "stage": ["src", "buildtool", "main"] , "deps": [ ["src/buildtool/common", "retry_cli"] , ["src/buildtool/common/remote", "retry_config"] ] , "private-deps": [ ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] } , "describe": { "type": ["@", "rules", "CC", "library"] , "name": ["describe"] , "hdrs": ["describe.hpp"] , "srcs": ["describe.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "buildtool", "main"] , "private-deps": [ "common" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "module_name"] , ["src/buildtool/build_engine/base_maps", "rule_map"] , ["src/buildtool/build_engine/base_maps", "targets_file_map"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] ] } , "common": { "type": ["@", "rules", "CC", "library"] , "name": ["common"] , "hdrs": ["exit_codes.hpp"] , "stage": ["src", "buildtool", "main"] } , "cli": { "type": ["@", "rules", "CC", "library"] , "name": ["cli"] , "hdrs": ["cli.hpp"] , "srcs": ["cli.cpp"] , "stage": ["src", "buildtool", "main"] , "deps": [["src/buildtool/common", "cli"], ["src/buildtool/common", "retry_cli"]] , "private-deps": [ "common" , ["@", "cli11", "", "cli11"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "install_cas": { "type": ["@", "rules", "CC", "library"] , "name": ["install_cas"] , "hdrs": ["install_cas.hpp"] , "srcs": ["install_cas.cpp"] , "deps": [ ["src/buildtool/common", "cli"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/remote", "context"] ] , "stage": ["src", "buildtool", "main"] , "private-deps": [ "archive" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/utils", "subobject"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] } , "add_to_cas": { "type": ["@", "rules", "CC", "library"] , "name": ["add_to_cas"] , "hdrs": ["add_to_cas.hpp"] , "srcs": ["add_to_cas.cpp"] , "deps": [ ["src/buildtool/common", "cli"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/storage", "storage"] ] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/file_system/symlinks", "resolve_special"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "buildtool", "main"] } , "analyse": { "type": ["@", "rules", "CC", "library"] , "name": ["analyse"] , "hdrs": ["analyse.hpp"] , "srcs": ["analyse.cpp"] , "deps": [ "analyse_context" , ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/target_map", "absent_target_map"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/profile", "profile"] ] , "stage": ["src", "buildtool", "main"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "directory_map"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/base_maps", "expression_map"] , ["src/buildtool/build_engine/base_maps", "rule_map"] , ["src/buildtool/build_engine/base_maps", "source_map"] , ["src/buildtool/build_engine/base_maps", "targets_file_map"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/common", "action_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/progress_reporting", "exports_progress_reporter"] , ["src/buildtool/storage", "storage"] ] } , "analyse_context": { "type": ["@", "rules", "CC", "library"] , "name": ["analyse_context"] , "hdrs": ["analyse_context.hpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "main"] } , "diagnose": { "type": ["@", "rules", "CC", "library"] , "name": ["diagnose"] , "hdrs": ["diagnose.hpp"] , "srcs": ["diagnose.cpp"] , "stage": ["src", "buildtool", "main"] , "deps": ["analyse", ["src/buildtool/common", "cli"]] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "json"] ] } , "version": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["SOURCE_DATE_EPOCH", "VERSION_EXTRA_SUFFIX"] , "name": ["version"] , "hdrs": ["version.hpp"] , "srcs": ["version.cpp"] , "private-defines": { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "SOURCE_DATE_EPOCH"} , "then": [ { "type": "join" , "$1": [ "SOURCE_DATE_EPOCH=" , { "type": "json_encode" , "$1": {"type": "var", "name": "SOURCE_DATE_EPOCH"} } ] } ] } , { "type": "if" , "cond": {"type": "var", "name": "VERSION_EXTRA_SUFFIX"} , "then": [ { "type": "join" , "$1": [ "VERSION_EXTRA_SUFFIX=" , { "type": "json_encode" , "$1": {"type": "var", "name": "VERSION_EXTRA_SUFFIX"} } ] } ] } ] } , "private-deps": [["@", "json", "", "json"], ["src/utils/cpp", "json"]] , "stage": ["src", "buildtool", "main"] } , "constants": { "type": ["@", "rules", "CC", "library"] , "name": ["constants"] , "hdrs": ["constants.hpp"] , "stage": ["src", "buildtool", "main"] } , "serve": { "type": ["@", "rules", "CC", "library"] , "name": ["serve"] , "hdrs": ["serve.hpp"] , "srcs": ["serve.cpp"] , "deps": ["cli", ["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "main"] , "private-deps": [ "build_utils" , "common" , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "location"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] } , "build_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["build_utils"] , "hdrs": ["build_utils.hpp"] , "srcs": ["build_utils.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "main"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] ] } , "archive": { "type": ["@", "rules", "CC", "library"] , "name": ["archive"] , "hdrs": ["archive.hpp"] , "srcs": ["archive.cpp"] , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "common"] ] , "stage": ["src", "buildtool", "main"] , "private-deps": [ ["", "libarchive"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/add_to_cas.cpp000066400000000000000000000451141516554100600261200ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/add_to_cas.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/symlinks/resolve_special.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/path_hash.hpp" namespace { /// \brief Class handling import of a filesystem directory to CAS. Allows /// various treatments of special entries (e.g., symlinks). class CASTreeImporter final { public: using FileStoreFunc = std::function< std::optional(std::filesystem::path const&, bool)>; using TreeStoreFunc = std::function(std::string const&)>; using SymlinkStoreFunc = std::function(std::string const&)>; using KnownPathsMapType = std::unordered_map>; explicit CASTreeImporter( std::filesystem::path const& root, FileStoreFunc store_file, TreeStoreFunc store_tree, SymlinkStoreFunc store_symlink, std::optional resolve_special = std::nullopt) noexcept : root_{ToNormalPath(std::filesystem::absolute(root))}, store_file_{std::move(store_file)}, store_tree_{std::move(store_tree)}, store_symlink_{std::move(store_symlink)}, resolve_special_{resolve_special} {}; /// \brief Get the Git-tree Digest of the directory, relative to the root. [[nodiscard]] auto GetDigest( std::filesystem::path const& relative_path = ".") noexcept -> std::optional { // cache already computed paths, to avoid extra work KnownPathsMapType known_paths; // to store directory paths pointed to by symlinks; this allows // detection of cycles when upward symlinks are involved std::unordered_set linked_trees; return this->CreateGitTreeDigest( relative_path, &known_paths, &linked_trees); } private: std::filesystem::path const root_; FileStoreFunc const store_file_; TreeStoreFunc const store_tree_; SymlinkStoreFunc const store_symlink_; std::optional const resolve_special_; [[nodiscard]] auto CreateGitTreeDigest( std::filesystem::path const& relative_path, gsl::not_null const& known_paths, gsl::not_null*> const& linked_trees) noexcept -> std::optional { try { // normalize path auto const dir = ToNormalPath(std::filesystem::absolute(root_ / relative_path)); // check the path is pointing to a directory if (not FileSystemManager::IsDirectory(dir)) { Logger::Log(LogLevel::Error, "Failed to store tree {} -- not a directory", dir.string()); return std::nullopt; } // check for cycles in resolving upwards symlinks if (not linked_trees->emplace(relative_path).second) { Logger::Log(LogLevel::Error, "Failed storing tree {} -- cycle found", dir.string()); return std::nullopt; } auto const remove_tree_from_set = gsl::finally([&linked_trees, &relative_path]() { linked_trees->erase(relative_path); }); // set up the directory reader lambda GitRepo::tree_entries_t entries{}; auto dir_reader = [this, &entries, &relative_path, &known_paths, &linked_trees]( std::filesystem::path const& name, ObjectType type) -> bool { auto rel_path_to_process = ToNormalPath(relative_path / name); auto full_path_to_process = ToNormalPath(root_ / rel_path_to_process); // check if entry is cached if (auto it = known_paths->find(rel_path_to_process); it != known_paths->end()) { if (auto raw_id = FromHexString(it->second.first.hash())) { entries[std::move(*raw_id)].emplace_back( name.string(), it->second.second); return true; } Logger::Log(LogLevel::Error, "Failed storing entry {}", full_path_to_process.string()); return false; } // original relative path to allow resolvable symlinks to also // be cached together with the entries they resolve to auto const rel_orig_path = rel_path_to_process; std::optional type_to_process = type; // process if symlink; do this first, as symlinks can resolve to // other types if (IsSymlinkObject(*type_to_process)) { // check if the symlink should be kept as-is if (not resolve_special_ or *resolve_special_ == ResolveSpecial::TreeUpwards) { // read symlink auto const content = FileSystemManager::ReadSymlink( full_path_to_process); if (not content) { Logger::Log(LogLevel::Error, "Failed reading symlink {}", full_path_to_process.string()); return false; } // if non-upwards, store symlink, cache path and set // entry if (PathIsNonUpwards(*content)) { if (auto digest = store_symlink_(*content)) { known_paths->emplace( rel_path_to_process, std::make_pair(*digest, ObjectType::Symlink)); if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back( name.string(), ObjectType::Symlink); return true; } } Logger::Log(LogLevel::Error, "Failed storing symlink {}", full_path_to_process.string()); return false; } // fail for non-upward symlink if on default behaviour if (not resolve_special_) { Logger::Log( LogLevel::Error, "Failed storing symlink {} -- not non-upwards", full_path_to_process.string()); return false; } } // resolve the symlink; do so in a loop in order to check, // depending on the resolve strategy, whether the resolve // chain ever goes outside the root tree; the resulting // entry can then be processed as its type std::unordered_set visited( {rel_path_to_process}); while (*type_to_process == ObjectType::Symlink) { // read symlink auto const content = FileSystemManager::ReadSymlink( full_path_to_process); if (not content) { Logger::Log(LogLevel::Error, "Failed reading symlink {}", full_path_to_process.string()); return false; } if ((*resolve_special_ == ResolveSpecial::TreeUpwards or *resolve_special_ == ResolveSpecial::TreeAll) and not PathIsConfined(*content, rel_path_to_process)) { Logger::Log(LogLevel::Error, "Failed resolving symlink {} -- not " "resolving inside root tree", full_path_to_process.string()); return false; } // follow the symlink full_path_to_process = ToNormalPath( full_path_to_process.parent_path() / *content); rel_path_to_process = full_path_to_process.lexically_relative(root_); type_to_process = FileSystemManager::Type(full_path_to_process, /*allow_upwards=*/true); if (not type_to_process) { Logger::Log(LogLevel::Error, "Failed getting type of entry {}", full_path_to_process.string()); return false; } // check for cycle while resolving if (not visited.emplace(rel_path_to_process).second) { Logger::Log( LogLevel::Error, "Failed resolving symlink {} -- cycle found", full_path_to_process.string()); return false; } } } // process if tree; can be initial entry or resolved symlink // under that name if (IsTreeObject(*type_to_process)) { // store tree and get digest if (auto digest = this->CreateGitTreeDigest( rel_path_to_process, known_paths, linked_trees)) { // cache the path to process, as well as the original // path (in case it is of a symlink) known_paths->emplace( rel_path_to_process, std::make_pair(*digest, ObjectType::Tree)); known_paths->emplace( rel_orig_path, std::make_pair(*digest, ObjectType::Tree)); // set entry if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back( name.string(), ObjectType::Tree); return true; } } Logger::Log(LogLevel::Error, "Failed storing tree {}", full_path_to_process.string()); return false; } // process if file; can be initial entry or resolved symlink // under that name if (IsFileObject(*type_to_process)) { // store file and get digest if (auto digest = store_file_(full_path_to_process, IsExecutableObject(*type_to_process))) { // cache the path to process, as well as the original // path (in case it is of a symlink) known_paths->emplace( rel_path_to_process, std::make_pair(*digest, *type_to_process)); known_paths->emplace( rel_orig_path, std::make_pair(*digest, *type_to_process)); // set entry if (auto raw_id = FromHexString(digest->hash())) { entries[std::move(*raw_id)].emplace_back( name.string(), *type_to_process); return true; } } Logger::Log(LogLevel::Error, "Failed storing file {}", full_path_to_process.string()); return false; } Logger::Log(LogLevel::Error, "Failed storing entry {} -- unsupported type", full_path_to_process.string()); return false; }; // read directory entries; skip any special entries if so configured if (FileSystemManager::ReadDirectory( dir, dir_reader, /*allow_upwards=*/true, /*ignore_special=*/resolve_special_ == ResolveSpecial::Ignore, /*log_failure_at=*/LogLevel::Error)) { if (auto tree = GitRepo::CreateShallowTree(entries)) { // store tree return store_tree_(tree->second); } } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Storing tree failed with:\n{}", ex.what()); } return std::nullopt; } }; } // namespace auto AddArtifactsToCas(ToAddArguments const& clargs, Storage const& storage, ApiBundle const& apis) -> bool { auto object_location = clargs.location; if (clargs.follow_symlinks) { if (not FileSystemManager::ResolveSymlinks(&object_location)) { Logger::Log(LogLevel::Error, "Failed resolving {}", clargs.location.string()); return false; } } auto object_type = FileSystemManager::Type(object_location, /*allow_upwards=*/true); if (not object_type) { Logger::Log(LogLevel::Error, "Non existent or unsupported file-system entry at {}", object_location.string()); return false; } auto const& cas = storage.CAS(); std::optional digest{}; switch (*object_type) { case ObjectType::File: digest = cas.StoreBlob(object_location, /*is_executable=*/false); break; case ObjectType::Executable: digest = cas.StoreBlob(object_location, /*is_executable=*/true); break; case ObjectType::Symlink: { auto content = FileSystemManager::ReadSymlink(object_location); if (not content) { Logger::Log(LogLevel::Error, "Failed to read symlink at {}", object_location.string()); return false; } digest = cas.StoreBlob(*content, /*is_executable=*/false); } break; case ObjectType::Tree: { if (not ProtocolTraits::IsTreeAllowed( cas.GetHashFunction().GetType())) { Logger::Log(LogLevel::Error, "Storing of trees only supported in native mode"); return false; } auto store_file = [&cas](std::filesystem::path const& path, auto is_exec) -> std::optional { return cas.StoreBlob(path, is_exec); }; auto store_tree = [&cas](std::string const& content) -> std::optional { return cas.StoreTree(content); }; auto store_symlink = [&cas](std::string const& content) -> std::optional { return cas.StoreBlob(content); }; CASTreeImporter tree_importer{object_location, store_file, store_tree, store_symlink, clargs.resolve_special}; digest = tree_importer.GetDigest(); } break; } if (not digest) { Logger::Log(LogLevel::Error, "Failed to store {} in local CAS", clargs.location.string()); return false; } std::cout << digest->hash() << std::endl; auto const object = std::vector{ Artifact::ObjectInfo{*digest, *object_type, false}}; if (not apis.local->RetrieveToCas(object, *apis.remote)) { Logger::Log(LogLevel::Error, "Failed to upload artifact to remote endpoint"); return false; } return true; } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/add_to_cas.hpp000066400000000000000000000022071516554100600261210ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_ADD_TO_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_ADD_TO_CAS_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/common/cli.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/storage/storage.hpp" [[nodiscard]] auto AddArtifactsToCas(ToAddArguments const& clargs, Storage const& storage, ApiBundle const& apis) -> bool; #endif #endif // INCLUDED_SRC_BUILDTOOL_MAIN_ADD_TO_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/analyse.cpp000066400000000000000000000306241516554100600254740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/analyse.hpp" #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include "src/buildtool/build_engine/base_maps/source_map.hpp" #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/progress_reporting/exports_progress_reporter.hpp" #include "src/buildtool/storage/storage.hpp" namespace { namespace Base = BuildMaps::Base; namespace Target = BuildMaps::Target; [[nodiscard]] auto GetActionNumber(const AnalysedTarget& target, int number) -> std::optional { auto const& actions = target.Actions(); if (number >= 0) { if (actions.size() > static_cast(number)) { return actions[static_cast(number)]; } } else { if (((static_cast(actions.size())) + static_cast(number)) >= 0) { return actions[static_cast( (static_cast(actions.size())) + static_cast(number))]; } } return std::nullopt; } [[nodiscard]] auto SwitchToActionInput(const AnalysedTargetPtr& target, const ActionDescription::Ptr& action) -> AnalysedTargetPtr { auto inputs = Expression::map_t::underlying_map_t{}; for (auto const& [k, v] : action->Inputs()) { inputs[k] = ExpressionPtr{Expression{v}}; } auto inputs_exp = ExpressionPtr{Expression::map_t{inputs}}; auto provides = nlohmann::json::object(); provides["cmd"] = action->GraphAction().Command(); provides["env"] = action->GraphAction().Env(); provides["output"] = action->OutputFiles(); provides["output_dirs"] = action->OutputDirs(); if (action->GraphAction().MayFail()) { provides["may_fail"] = *(action->GraphAction().MayFail()); } if (action->GraphAction().NoCache()) { provides["no_cache"] = true; } if (action->GraphAction().TimeoutScale() != 1.0) { provides["timeout scaling"] = action->GraphAction().TimeoutScale(); } if (not action->GraphAction().Cwd().empty()) { provides["cwd"] = action->GraphAction().Cwd(); } if (not action->GraphAction().ExecutionProperties().empty()) { provides["execution properties"] = action->GraphAction().ExecutionProperties(); } auto provides_exp = Expression::FromJson(provides); return std::make_shared( TargetResult{.artifact_stage = inputs_exp, .provides = provides_exp, .runfiles = Expression::kEmptyMap}, std::vector{action}, target->Blobs(), target->Trees(), target->TreeOverlays(), target->Vars(), target->Tainted(), target->ImpliedExport(), target->GraphInformation()); } } // namespace [[nodiscard]] auto AnalyseTarget( gsl::not_null const& context, const Target::ConfiguredTarget& id, std::size_t jobs, std::optional const& request_action_input, Logger const* logger, BuildMaps::Target::ServeFailureLogReporter* serve_log, Profile* profile) -> std::optional { // create async maps auto directory_entries = Base::CreateDirectoryEntriesMap(context->repo_config, jobs); auto expressions_file_map = Base::CreateExpressionFileMap(context->repo_config, jobs); auto rule_file_map = Base::CreateRuleFileMap(context->repo_config, jobs); auto targets_file_map = Base::CreateTargetsFileMap(context->repo_config, jobs); auto expr_map = Base::CreateExpressionMap( &expressions_file_map, context->repo_config, jobs); auto rule_map = Base::CreateRuleMap( &rule_file_map, &expr_map, context->repo_config, jobs); auto source_targets = Base::CreateSourceTargetMap( &directory_entries, context->repo_config, context->storage->GetHashFunction().GetType(), jobs); auto absent_target_variables_map = Target::CreateAbsentTargetVariablesMap(context, jobs); BuildMaps::Target::ResultTargetMap result_map{jobs}; auto absent_target_map = Target::CreateAbsentTargetMap( context, &result_map, &absent_target_variables_map, jobs, serve_log); auto target_map = Target::CreateTargetMap(context, &source_targets, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map, jobs); Logger::Log( logger, LogLevel::Info, "Requested target is {}", id.ToString()); AnalysedTargetPtr target{}; // we should only report served export targets if a serve endpoint exists bool const has_serve = context->serve != nullptr; auto reporter = ExportsProgressReporter::Reporter( context->statistics, context->progress, has_serve, logger); std::atomic done{false}; std::condition_variable cv{}; auto observer = std::thread([reporter, &done, &cv]() { reporter(&done, &cv); }); bool failed{false}; { TaskSystem ts{jobs}; target_map.ConsumeAfterKeysReady( &ts, {id}, [&target](auto values) { target = *values[0]; }, [&failed, logger, profile](auto const& msg, bool fatal) { Logger::Log(logger, fatal ? LogLevel::Error : LogLevel::Warning, "While processing targets:\n{}", msg); failed = failed or fatal; if (fatal and (profile != nullptr)) { profile->NoteAnalysisError(msg); } }); } // close analysis progress observer done = true; cv.notify_all(); observer.join(); if (failed) { return std::nullopt; } if (not target) { Logger::Log(logger, LogLevel::Error, "Failed to analyse target: {}", id.ToString()); if (auto error_msg = DetectAndReportCycle( "expression imports", expr_map, Base::kEntityNamePrinter)) { Logger::Log(logger, LogLevel::Error, *error_msg); return std::nullopt; } if (auto error_msg = DetectAndReportCycle("target dependencies", target_map, Target::kConfiguredTargetPrinter)) { Logger::Log(logger, LogLevel::Error, *error_msg); return std::nullopt; } DetectAndReportPending( "expressions", expr_map, Base::kEntityNamePrinter, logger); DetectAndReportPending( "rules", rule_map, Base::kEntityNamePrinter, logger); DetectAndReportPending( "targets", target_map, Target::kConfiguredTargetPrinter, logger); return std::nullopt; } // Clean up in parallel what is no longer needed { TaskSystem ts{jobs}; target_map.Clear(&ts); source_targets.Clear(&ts); directory_entries.Clear(&ts); expressions_file_map.Clear(&ts); rule_file_map.Clear(&ts); targets_file_map.Clear(&ts); expr_map.Clear(&ts); rule_map.Clear(&ts); } std::optional modified{}; if (request_action_input) { if (request_action_input->starts_with("%")) { auto action_id = request_action_input->substr(1); auto action = result_map.GetAction(action_id); if (action) { Logger::Log(logger, LogLevel::Info, "Request is input of action %{}", action_id); target = SwitchToActionInput(target, *action); modified = fmt::format("%{}", action_id); } else { Logger::Log(logger, LogLevel::Error, "Action {} not part of the action graph of the " "requested target", action_id); return std::nullopt; } } else if (request_action_input->starts_with("#")) { auto number = std::atoi(request_action_input->substr(1).c_str()); auto action = GetActionNumber(*target, number); if (action) { Logger::Log(logger, LogLevel::Info, "Request is input of action #{}", number); target = SwitchToActionInput(target, *action); modified = fmt::format("#{}", number); } else { Logger::Log(logger, LogLevel::Error, "Action #{} out of range for the requested target", number); return std::nullopt; } } else { auto action = result_map.GetAction(*request_action_input); if (action) { Logger::Log(logger, LogLevel::Info, "Request is input of action %{}", *request_action_input); target = SwitchToActionInput(target, *action); modified = fmt::format("%{}", *request_action_input); } else { auto number = std::atoi(request_action_input->c_str()); auto action = GetActionNumber(*target, number); if (action) { Logger::Log(logger, LogLevel::Info, "Request is input of action #{}", number); target = SwitchToActionInput(target, *action); modified = fmt::format("#{}", number); } else { Logger::Log( logger, LogLevel::Error, "Action #{} out of range for the requested target", number); return std::nullopt; } } } } return AnalysisResult{.id = id, .target = target, .result_map = std::move(result_map), .modified = modified}; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/analyse.hpp000066400000000000000000000034441516554100600255010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_HPP #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/profile/profile.hpp" struct AnalysisResult { BuildMaps::Target::ConfiguredTarget id; AnalysedTargetPtr target; BuildMaps::Target::ResultTargetMap result_map; std::optional modified; }; [[nodiscard]] auto AnalyseTarget( gsl::not_null const& context, const BuildMaps::Target::ConfiguredTarget& id, std::size_t jobs, std::optional const& request_action_input, Logger const* logger = nullptr, BuildMaps::Target::ServeFailureLogReporter* = nullptr, Profile* = nullptr) -> std::optional; #endif // INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/analyse_context.hpp000066400000000000000000000027051516554100600272440ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_CONTEXT_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_CONTEXT_HPP #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/storage.hpp" /// \brief Aggregate to be passed during analysis. /// \note No field is stored as const ref to avoid binding to temporaries. struct AnalyseContext final { gsl::not_null const repo_config; gsl::not_null const storage; gsl::not_null const statistics; gsl::not_null const progress; ServeApi const* const serve = nullptr; }; #endif // INCLUDED_SRC_BUILDOOL_MAIN_ANALYSE_CONTEXT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/archive.cpp000066400000000000000000000201061516554100600254530ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/main/archive.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" extern "C" { #include #include } namespace { void archive_write_closer(archive* a) { if (a != nullptr) { archive_write_close(a); archive_write_free(a); } } void archive_entry_cleanup(archive_entry* entry) { if (entry != nullptr) { archive_entry_free(entry); } } auto add_to_archive(archive* archive, IExecutionApi const& api, const Artifact::ObjectInfo& artifact, const std::filesystem::path& location) -> bool { auto constexpr kExecutablePerm = 0555; auto constexpr kFilePerm = 0444; auto constexpr kDefaultPerm = 07777; auto payload = api.RetrieveToMemory(artifact); if (not payload) { Logger::Log(LogLevel::Error, "Failed to retrieve artifact {}", artifact.ToString()); return false; } switch (artifact.type) { case ObjectType::File: case ObjectType::Executable: { std::unique_ptr entry{archive_entry_new(), archive_entry_cleanup}; archive_entry_set_pathname(entry.get(), location.string().c_str()); archive_entry_set_size(entry.get(), static_cast(payload->size())); archive_entry_set_filetype(entry.get(), AE_IFREG); archive_entry_set_perm(entry.get(), artifact.type == ObjectType::Executable ? kExecutablePerm : kFilePerm); archive_write_header(archive, entry.get()); archive_write_data(archive, payload->c_str(), payload->size()); } break; case ObjectType::Symlink: { std::unique_ptr entry{archive_entry_new(), archive_entry_cleanup}; archive_entry_set_pathname(entry.get(), location.string().c_str()); archive_entry_set_size(entry.get(), static_cast(payload->size())); archive_entry_set_filetype(entry.get(), AE_IFLNK); archive_entry_set_symlink(entry.get(), payload->c_str()); archive_entry_set_perm(entry.get(), kDefaultPerm); archive_write_header(archive, entry.get()); archive_write_data(archive, payload->c_str(), payload->size()); } break; case ObjectType::Tree: { // avoid creating empty unnamed folder for the initial call if (not location.empty()) { std::unique_ptr entry{archive_entry_new(), archive_entry_cleanup}; archive_entry_set_pathname(entry.get(), location.string().c_str()); archive_entry_set_size(entry.get(), 0U); archive_entry_set_filetype(entry.get(), AE_IFDIR); archive_entry_set_perm(entry.get(), kDefaultPerm); archive_write_header(archive, entry.get()); } auto git_tree = GitRepo::ReadTreeData( *payload, artifact.digest.hash(), [](auto const& /*unused*/) { return true; }, /*is_hex_id=*/true); if (not git_tree) { Logger::Log(LogLevel::Error, "Failed to parse {} as git tree for path {}", artifact.ToString(), location.string()); return false; } // Reorder entries to be keyed and sorted by name std::map tree{}; for (auto const& [hash, entries] : *git_tree) { auto hex_hash = ToHexString(hash); for (auto const& entry : entries) { auto digest = ArtifactDigestFactory::Create(api.GetHashType(), hex_hash, 0, IsTreeObject(entry.type)); if (not digest) { return false; } tree[entry.name] = Artifact::ObjectInfo{.digest = *std::move(digest), .type = entry.type, .failed = false}; } } for (auto const& [name, obj] : tree) { if (not add_to_archive(archive, api, obj, location / name)) { return false; } } } break; } return true; } } // namespace [[nodiscard]] auto GenerateArchive( IExecutionApi const& api, const Artifact::ObjectInfo& artifact, const std::optional& output_path) -> bool { constexpr int kTarBlockSize = 512; std::unique_ptr archive{ archive_write_new(), archive_write_closer}; if (archive == nullptr) { Logger::Log(LogLevel::Error, "Internal error: Call to archive_write_new() failed"); return false; } if (archive_write_set_format_pax_restricted(archive.get()) != ARCHIVE_OK) { Logger::Log(LogLevel::Error, "Internal error: Call to " "archive_write_set_format_pax_restriced() failed"); return false; } if (archive_write_set_bytes_per_block(archive.get(), kTarBlockSize) != ARCHIVE_OK) { Logger::Log(LogLevel::Error, "Internal error: Call to " "archive_write_set_bytes_per_block() failed"); return false; } if (output_path) { if (archive_write_open_filename( archive.get(), output_path->string().c_str()) != ARCHIVE_OK) { Logger::Log(LogLevel::Error, "Failed to open archive for writing at {}", output_path->string()); return false; } } else { if (archive_write_open_fd(archive.get(), STDOUT_FILENO) != ARCHIVE_OK) { Logger::Log(LogLevel::Error, "Failed to open stdout for writing archive to"); return false; } } if (not add_to_archive( archive.get(), api, artifact, std::filesystem::path{""})) { return false; } if (output_path) { Logger::Log(LogLevel::Info, "Archive of {} was installed to {}", artifact.ToString(), output_path->string()); } return true; } #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/archive.hpp000066400000000000000000000021631516554100600254630ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_ARCHIVE_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_ARCHIVE_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" [[nodiscard]] auto GenerateArchive( IExecutionApi const& api, const Artifact::ObjectInfo& artifact, const std::optional& output_path) -> bool; #endif #endif // INCLUDED_SRC_BUILDTOOL_MAIN_ARCHIVE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/build_utils.cpp000066400000000000000000000205321516554100600263540ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/build_utils.hpp" #include #include #include #include #include "fmt/core.h" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #endif // BOOTSTRAP_BUILD_TOOL auto ReadOutputArtifacts(AnalysedTargetPtr const& target) -> std::pair, std::map> { std::map artifacts{}; std::map runfiles{}; for (auto const& [path, artifact] : target->Artifacts()->Map()) { artifacts.emplace(path, artifact->Artifact()); } for (auto const& [path, artifact] : target->RunFiles()->Map()) { if (not artifacts.contains(path)) { runfiles.emplace(path, artifact->Artifact()); } } return {artifacts, runfiles}; } auto CollectNonKnownArtifacts( std::unordered_map const& cache_targets) -> std::vector { auto cache_artifacts = std::unordered_set{}; for (auto const& [_, target] : cache_targets) { auto artifacts = target->ContainedNonKnownArtifacts(); cache_artifacts.insert(std::make_move_iterator(artifacts.begin()), std::make_move_iterator(artifacts.end())); } return {std::make_move_iterator(cache_artifacts.begin()), std::make_move_iterator(cache_artifacts.end())}; } auto ToTargetCacheWriteStrategy(std::string const& strategy) -> std::optional { if (strategy == "disable") { return TargetCacheWriteStrategy::Disable; } if (strategy == "sync") { return TargetCacheWriteStrategy::Sync; } if (strategy == "split") { return TargetCacheWriteStrategy::Split; } return std::nullopt; } #ifndef BOOTSTRAP_BUILD_TOOL auto CreateTargetCacheWriterMap( std::unordered_map const& cache_targets, std::unordered_map const& extra_infos, std::size_t jobs, gsl::not_null const& apis, TargetCacheWriteStrategy strategy, TargetCache const& tc) -> TargetCacheWriterMap { auto write_tc_entry = [cache_targets, extra_infos, jobs, apis, strategy, tc]( auto /*ts*/, auto setter, auto logger, auto subcaller, auto const& key) { // get the TaretCacheKey corresponding to this Id TargetCacheKey tc_key{key}; // check if entry actually needs storing auto const tc_key_it = cache_targets.find(tc_key); if (tc_key_it == cache_targets.end()) { if (tc.Read(tc_key)) { // entry already in target-cache, so nothing to be done (*setter)(nullptr); return; } (*logger)(fmt::format("Export target {} not analysed locally; " "not caching anything depending on it", key.ToString()), true); return; } auto entry = TargetCacheEntry::FromTarget( apis->remote->GetHashType(), tc_key_it->second, extra_infos); if (not entry) { (*logger)( fmt::format("Failed creating target cache entry for key {}", key.ToString()), /*fatal=*/true); return; } // only store current entry once all implied targets are stored if (auto implied_targets = entry->ToImpliedIds(key.digest.hash())) { (*subcaller)( *implied_targets, [tc_key, entry, jobs, apis, strategy, tc, setter, logger]( [[maybe_unused]] auto const& values) { // create parallel artifacts downloader auto downloader = [apis, &jobs, strategy](auto infos) { return apis->remote->ParallelRetrieveToCas( infos, *apis->local, jobs, strategy == TargetCacheWriteStrategy::Split); }; if (not tc.Store(tc_key, *entry, downloader)) { (*logger)(fmt::format("Failed writing target cache " "entry for {}", tc_key.Id().ToString()), /*fatal=*/true); return; } // success! (*setter)(nullptr); }, logger); } else { (*logger)(fmt::format("Failed retrieving implied targets for " "key {}", key.ToString()), /*fatal=*/true); } }; return AsyncMapConsumer( write_tc_entry, jobs); } void WriteTargetCacheEntries( std::unordered_map const& cache_targets, std::unordered_map const& extra_infos, std::size_t jobs, ApiBundle const& apis, TargetCacheWriteStrategy strategy, TargetCache const& tc, Logger const* logger, LogLevel log_level) { std::size_t sqrt_jobs = std::lround(std::ceil(std::sqrt(jobs))); if (strategy == TargetCacheWriteStrategy::Disable) { return; } if (not cache_targets.empty()) { Logger::Log(logger, LogLevel::Info, "Backing up artifacts of {} export targets", cache_targets.size()); } // set up writer map auto tc_writer_map = CreateTargetCacheWriterMap( cache_targets, extra_infos, sqrt_jobs, &apis, strategy, tc); std::vector cache_targets_ids; cache_targets_ids.reserve(cache_targets.size()); for (auto const& [k, _] : cache_targets) { cache_targets_ids.emplace_back(k.Id()); } // write the target cache keys bool failed{false}; { TaskSystem ts{sqrt_jobs}; tc_writer_map.ConsumeAfterKeysReady( &ts, cache_targets_ids, []([[maybe_unused]] auto _) {}, // map doesn't set anything [&failed, logger, log_level](auto const& msg, bool fatal) { Logger::Log(logger, log_level, "While writing target cache entries:\n{}", msg); failed = failed or fatal; }); } // check for failures and cycles if (failed) { return; } if (auto error = DetectAndReportCycle( "writing cache targets", tc_writer_map, kObjectInfoPrinter)) { Logger::Log(logger, log_level, *error); return; } Logger::Log(logger, LogLevel::Debug, "Finished backing up artifacts of export targets"); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/build_utils.hpp000066400000000000000000000075551516554100600263730ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_BUILD_UTILS_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_BUILD_UTILS_HPP #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/storage/target_cache.hpp" #endif // BOOTSTRAP_BUILD_TOOL // Return disjoint maps for artifacts and runfiles [[nodiscard]] auto ReadOutputArtifacts(AnalysedTargetPtr const& target) -> std::pair, std::map>; // Collect non-known artifacts [[nodiscard]] auto CollectNonKnownArtifacts( std::unordered_map const& cache_targets) -> std::vector; enum class TargetCacheWriteStrategy : std::uint8_t { Disable, ///< Do not create target-level cache entries Sync, ///< Create target-level cache entries after syncing the artifacts Split ///< Create target-level cache entries after syncing the artifacts; ///< during artifact sync try to use blob splitting, if available }; auto ToTargetCacheWriteStrategy(std::string const&) -> std::optional; #ifndef BOOTSTRAP_BUILD_TOOL /// \brief Maps the Id of a TargetCacheKey to nullptr_t, as we only care if /// writing the tc entry succeeds or not. using TargetCacheWriterMap = AsyncMapConsumer; /// \brief Handles the writing of target cache keys after analysis concludes. [[nodiscard]] auto CreateTargetCacheWriterMap( std::unordered_map const& cache_targets, std::unordered_map const& extra_infos, std::size_t jobs, gsl::not_null const& apis, TargetCacheWriteStrategy strategy, TargetCache const& tc) -> TargetCacheWriterMap; // use explicit cast to std::function to allow template deduction when used static const std::function kObjectInfoPrinter = [](Artifact::ObjectInfo const& x) -> std::string { return x.ToString(); }; /// \brief Write the target cache entries resulting after a build. /// \param log_level the level at which to report failure void WriteTargetCacheEntries( std::unordered_map const& cache_targets, std::unordered_map const& extra_infos, std::size_t jobs, ApiBundle const& apis, TargetCacheWriteStrategy strategy, TargetCache const& tc, Logger const* logger = nullptr, LogLevel log_level = LogLevel::Warning); #endif // BOOTSTRAP_BUILD_TOOL #endif // INCLUDED_SRC_BUILDOOL_MAIN_BUILD_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/cli.cpp000066400000000000000000000253321516554100600246070ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/cli.hpp" #include #include #include "CLI/CLI.hpp" #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/exit_codes.hpp" namespace { /// \brief Setup arguments for sub command "just describe". auto SetupDescribeCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupCommonArguments(app, &clargs->common); SetupCacheArguments(app, &clargs->endpoint); SetupAnalysisArguments(app, &clargs->analysis, false); SetupLogArguments(app, &clargs->log); SetupServeEndpointArguments(app, &clargs->serve); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupProtocolArguments(app, &clargs->protocol); SetupDescribeArguments(app, &clargs->describe); SetupRetryArguments(app, &clargs->retry); } /// \brief Setup arguments for sub command "just analyse". auto SetupAnalyseCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupCommonArguments(app, &clargs->common); SetupLogArguments(app, &clargs->log); SetupAnalysisArguments(app, &clargs->analysis); SetupCacheArguments(app, &clargs->endpoint); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupExecutionPropertiesArguments(app, &clargs->endpoint); SetupServeEndpointArguments(app, &clargs->serve); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupCommonBuildArguments(app, &clargs->build); SetupBuildArguments(app, &clargs->build); SetupDiagnosticArguments(app, &clargs->diagnose); SetupProtocolArguments(app, &clargs->protocol); SetupRetryArguments(app, &clargs->retry); } /// \brief Setup arguments for sub command "just build". auto SetupBuildCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupCommonArguments(app, &clargs->common); SetupLogArguments(app, &clargs->log); SetupAnalysisArguments(app, &clargs->analysis); SetupCacheArguments(app, &clargs->endpoint); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupExecutionPropertiesArguments(app, &clargs->endpoint); SetupServeEndpointArguments(app, &clargs->serve); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupCommonBuildArguments(app, &clargs->build); SetupBuildArguments(app, &clargs->build); SetupExtendedBuildArguments(app, &clargs->build); SetupTCArguments(app, &clargs->tc); SetupProtocolArguments(app, &clargs->protocol); SetupRetryArguments(app, &clargs->retry); } /// \brief Setup arguments for sub command "just install". auto SetupInstallCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupBuildCommandArguments(app, clargs); // same as build SetupStageArguments(app, &clargs->stage); // plus stage } /// \brief Setup arguments for sub command "just rebuild". auto SetupRebuildCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupBuildCommandArguments(app, clargs); // same as build SetupRebuildArguments(app, &clargs->rebuild); // plus rebuild } /// \brief Setup arguments for sub command "just install-cas". auto SetupInstallCasCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupProtocolArguments(app, &clargs->protocol); SetupCacheArguments(app, &clargs->endpoint); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupFetchArguments(app, &clargs->fetch); SetupLogArguments(app, &clargs->log); SetupRetryArguments(app, &clargs->retry); } /// \brief Setup arguments for sub command "just install-cas". auto SetupAddToCasCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupProtocolArguments(app, &clargs->protocol); SetupCacheArguments(app, &clargs->endpoint); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupLogArguments(app, &clargs->log); SetupRetryArguments(app, &clargs->retry); SetupToAddArguments(app, &clargs->to_add); } /// \brief Setup arguments for sub command "just traverse". auto SetupTraverseCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupCommonArguments(app, &clargs->common); SetupLogArguments(app, &clargs->log); SetupCacheArguments(app, &clargs->endpoint); SetupExecutionEndpointArguments(app, &clargs->endpoint); SetupExecutionPropertiesArguments(app, &clargs->endpoint); SetupCommonAuthArguments(app, &clargs->auth); SetupClientAuthArguments(app, &clargs->cauth); SetupGraphArguments(app, &clargs->graph); // instead of analysis SetupCommonBuildArguments(app, &clargs->build); SetupBuildArguments(app, &clargs->build); SetupExtendedBuildArguments(app, &clargs->build); SetupStageArguments(app, &clargs->stage); SetupProtocolArguments(app, &clargs->protocol); } /// \brief Setup arguments for sub command "just gc". auto SetupGcCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupLogArguments(app, &clargs->log); SetupCacheArguments(app, &clargs->endpoint); SetupGcArguments(app, &clargs->gc); } /// \brief Setup arguments for sub command "just execute". auto SetupExecutionServiceCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupProtocolArguments(app, &clargs->protocol); SetupCommonBuildArguments(app, &clargs->build); SetupCacheArguments(app, &clargs->endpoint); SetupServiceArguments(app, &clargs->service); SetupLogArguments(app, &clargs->log); SetupCommonAuthArguments(app, &clargs->auth); SetupServerAuthArguments(app, &clargs->sauth); } /// \brief Setup arguments for sub command "just serve". auto SetupServeServiceCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { // all other arguments will be read from config file SetupServeArguments(app, &clargs->serve); } } // namespace auto ParseCommandLineArguments(int argc, char const* const* argv) -> CommandLineArguments { CLI::App app("just, a generic build tool"); app.option_defaults()->take_last(); auto* cmd_version = app.add_subcommand( "version", "Print version information in JSON format."); auto* cmd_describe = app.add_subcommand( "describe", "Describe the rule generating a target."); auto* cmd_analyse = app.add_subcommand("analyse", "Analyse specified targets."); auto* cmd_build = app.add_subcommand("build", "Build specified targets."); auto* cmd_install = app.add_subcommand("install", "Build and stage specified targets."); auto* cmd_rebuild = app.add_subcommand( "rebuild", "Rebuild and compare artifacts to cached build."); auto* cmd_install_cas = app.add_subcommand("install-cas", "Fetch and stage artifact from CAS."); auto* cmd_add_to_cas = app.add_subcommand( "add-to-cas", "Add a local file or directory to CAS."); auto* cmd_gc = app.add_subcommand("gc", "Trigger garbage collection of local cache."); auto* cmd_execution = app.add_subcommand( "execute", "Start single node execution service on this machine."); auto* cmd_serve = app.add_subcommand("serve", "Provide target dependencies for a build."); auto* cmd_traverse = app.group("") // group for creating hidden options ->add_subcommand("traverse", "Build and stage artifacts from graph file."); app.require_subcommand(1); CommandLineArguments clargs; SetupDescribeCommandArguments(cmd_describe, &clargs); SetupAnalyseCommandArguments(cmd_analyse, &clargs); SetupBuildCommandArguments(cmd_build, &clargs); SetupInstallCommandArguments(cmd_install, &clargs); SetupRebuildCommandArguments(cmd_rebuild, &clargs); SetupInstallCasCommandArguments(cmd_install_cas, &clargs); SetupAddToCasCommandArguments(cmd_add_to_cas, &clargs); SetupTraverseCommandArguments(cmd_traverse, &clargs); SetupGcCommandArguments(cmd_gc, &clargs); SetupExecutionServiceCommandArguments(cmd_execution, &clargs); SetupServeServiceCommandArguments(cmd_serve, &clargs); try { app.parse(argc, argv); } catch (CLI::Error& e) { auto exit_code = app.exit(e); std::exit(exit_code == 0 ? kExitSuccess : kExitSyntaxError); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Command line parse error: {}", ex.what()); std::exit(kExitSyntaxError); } if (*cmd_version) { clargs.cmd = SubCommand::kVersion; } else if (*cmd_describe) { clargs.cmd = SubCommand::kDescribe; } else if (*cmd_analyse) { clargs.cmd = SubCommand::kAnalyse; } else if (*cmd_build) { clargs.cmd = SubCommand::kBuild; } else if (*cmd_install) { clargs.cmd = SubCommand::kInstall; } else if (*cmd_rebuild) { clargs.cmd = SubCommand::kRebuild; } else if (*cmd_install_cas) { clargs.cmd = SubCommand::kInstallCas; } else if (*cmd_add_to_cas) { clargs.cmd = SubCommand::kAddToCas; } else if (*cmd_traverse) { clargs.cmd = SubCommand::kTraverse; } else if (*cmd_gc) { clargs.cmd = SubCommand::kGc; } else if (*cmd_execution) { clargs.cmd = SubCommand::kExecute; } else if (*cmd_serve) { clargs.cmd = SubCommand::kServe; } return clargs; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/cli.hpp000066400000000000000000000035031516554100600246100ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_CLI #define INCLUDED_SRC_BUILDTOOL_MAIN_CLI #include #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/retry_cli.hpp" enum class SubCommand : std::uint8_t { kUnknown, kVersion, kDescribe, kAnalyse, kBuild, kInstall, kRebuild, kInstallCas, kAddToCas, kTraverse, kGc, kExecute, kServe }; struct CommandLineArguments { CommonArguments common; LogArguments log; AnalysisArguments analysis; DescribeArguments describe; DiagnosticArguments diagnose; EndpointArguments endpoint; BuildArguments build; TCArguments tc; StageArguments stage; RebuildArguments rebuild; FetchArguments fetch; GraphArguments graph; CommonAuthArguments auth; ClientAuthArguments cauth; ServerAuthArguments sauth; ServiceArguments service; ServeArguments serve; RetryArguments retry; GcArguments gc; ToAddArguments to_add; ProtocolArguments protocol; SubCommand cmd{SubCommand::kUnknown}; }; auto ParseCommandLineArguments(int argc, char const* const* argv) -> CommandLineArguments; #endif // INCLUDED_SRC_BUILDTOOL_MAIN_CLI just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/constants.hpp000066400000000000000000000015641516554100600260620ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_CONSTANTS_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_CONSTANTS_HPP #include #include std::vector const kRootMarkers{".git", "ROOT", "WORKSPACE"}; #endif // INCLUDED_SRC_BUILDOOL_MAIN_CONSTANTS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/describe.cpp000066400000000000000000000452571516554100600256300ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/main/describe.hpp" #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/exit_codes.hpp" #include "src/buildtool/multithreading/task_system.hpp" namespace { namespace Base = BuildMaps::Base; void PrintDoc(const nlohmann::json& doc, const std::string& indent) { if (not doc.is_array()) { return; } for (auto const& line : doc) { if (line.is_string()) { std::cout << indent << line.get() << "\n"; } } } void PrintFields(nlohmann::json const& fields, const nlohmann::json& fdoc, const std::string& indent_field, const std::string& indent_field_doc) { for (auto const& f : fields) { std::cout << indent_field << f << "\n"; auto doc = fdoc.find(f); if (doc != fdoc.end()) { PrintDoc(*doc, indent_field_doc); } } } void PrettyPrintRule( nlohmann::json const& rdesc, BuildMaps::Base::EntityName const& rule_name, gsl::not_null const& repo_config) { auto doc = rdesc.find("doc"); if (doc != rdesc.end()) { PrintDoc(*doc, " | "); } auto field_doc = nlohmann::json::object(); auto field_doc_it = rdesc.find("field_doc"); if (field_doc_it != rdesc.end() and field_doc_it->is_object()) { field_doc = *field_doc_it; } auto string_fields = rdesc.find("string_fields"); if (string_fields != rdesc.end() and (not string_fields->empty())) { std::cout << " String fields\n"; PrintFields(*string_fields, field_doc, " - ", " | "); } auto target_fields = rdesc.find("target_fields"); if (target_fields != rdesc.end() and (not target_fields->empty())) { std::cout << " Target fields\n"; PrintFields(*target_fields, field_doc, " - ", " | "); } auto implicit_targets = rdesc.find("implicit"); if (implicit_targets != rdesc.end()) { for (auto const& [key, value] : implicit_targets->items()) { std::cout << " - implicit dependency\n"; // auto doc = field_doc.find(key); if (doc != field_doc.end()) { PrintDoc(*doc, " | "); } for (auto const& entry : value) { auto resolved_entry = BuildMaps::Base::ParseEntityNameFromJson( entry, rule_name, repo_config, [&entry, &rule_name](std::string const& parse_err) { Logger::Log(LogLevel::Warning, "Failed to resolve {} relative to {}:\n{}", entry.dump(), rule_name.ToString(), parse_err); }); if (resolved_entry) { std::cout << " - " << resolved_entry->ToString() << "\n"; } } } } auto config_fields = rdesc.find("config_fields"); if (config_fields != rdesc.end() and (not config_fields->empty())) { std::cout << " Config fields\n"; PrintFields(*config_fields, field_doc, " - ", " | "); } auto config_doc = nlohmann::json::object(); auto config_doc_it = rdesc.find("config_doc"); if (config_doc_it != rdesc.end() and config_doc_it->is_object()) { config_doc = *config_doc_it; } auto config_vars = rdesc.find("config_vars"); if (config_vars != rdesc.end() and (not config_vars->empty())) { std::cout << " Variables taken from the configuration\n"; PrintFields(*config_vars, config_doc, " - ", " | "); } std::cout << " Result\n"; std::cout << " - Artifacts\n"; auto artifacts_doc = rdesc.find("artifacts_doc"); if (artifacts_doc != rdesc.end()) { PrintDoc(*artifacts_doc, " | "); } std::cout << " - Runfiles\n"; auto runfiles_doc = rdesc.find("runfiles_doc"); if (runfiles_doc != rdesc.end()) { PrintDoc(*runfiles_doc, " | "); } auto provides_doc = rdesc.find("provides_doc"); if (provides_doc != rdesc.end()) { std::cout << " - Documented providers\n"; for (auto const& el : provides_doc->items()) { std::cout << " - " << el.key() << "\n"; PrintDoc(el.value(), " | "); } } std::cout << std::endl; } void PrintRuleAsOrderedJson(nlohmann::json const& rdesc, nlohmann::json const& rule_name) { auto string_fields = nlohmann::json::array(); if (auto doc = rdesc.find("string_fields"); doc != rdesc.end()) { string_fields = *doc; } auto target_fields = nlohmann::json::array(); if (auto doc = rdesc.find("target_fields"); doc != rdesc.end()) { target_fields = *doc; } auto config_fields = nlohmann::json::array(); if (auto doc = rdesc.find("config_fields"); doc != rdesc.end()) { config_fields = *doc; } auto config_vars = nlohmann::json::array(); if (auto doc = rdesc.find("config_vars"); doc != rdesc.end()) { config_vars = *doc; } auto field_doc = nlohmann::ordered_json::object(); if (auto doc = rdesc.find("field_doc"); doc != rdesc.end() and doc->is_object()) { auto fields = nlohmann::json::array(); fields.insert(fields.end(), string_fields.begin(), string_fields.end()); fields.insert(fields.end(), target_fields.begin(), target_fields.end()); fields.insert(fields.end(), config_fields.begin(), config_fields.end()); for (auto const& field : fields) { if (doc->contains(field)) { auto name = field.get(); field_doc[name] = (*doc)[name]; } } } auto config_doc = nlohmann::ordered_json::object(); if (auto doc = rdesc.find("config_doc"); doc != rdesc.end() and doc->is_object()) { for (auto const& var : config_vars) { if (doc->contains(var)) { auto name = var.get(); config_doc[name] = (*doc)[name]; } } } auto json_doc = nlohmann::ordered_json::object(); json_doc["type"] = rule_name; if (auto doc = rdesc.find("doc"); doc != rdesc.end()) { json_doc["doc"] = *doc; } if (not string_fields.empty()) { json_doc["string_fields"] = string_fields; } if (not target_fields.empty()) { json_doc["target_fields"] = target_fields; } if (not config_fields.empty()) { json_doc["config_fields"] = config_fields; } if (not field_doc.empty()) { json_doc["field_doc"] = field_doc; } if (not config_vars.empty()) { json_doc["config_vars"] = config_vars; } if (not config_doc.empty()) { json_doc["config_doc"] = config_doc; } if (auto doc = rdesc.find("artifacts_doc"); doc != rdesc.end()) { json_doc["artifacts_doc"] = *doc; } if (auto doc = rdesc.find("runfiles_doc"); doc != rdesc.end()) { json_doc["runfiles_doc"] = *doc; } if (auto doc = rdesc.find("provides_doc"); doc != rdesc.end()) { json_doc["provides_doc"] = *doc; } std::cout << json_doc.dump(2) << std::endl; } } // namespace auto DescribeUserDefinedRule( BuildMaps::Base::EntityName const& rule_name, gsl::not_null const& repo_config, std::size_t jobs, bool print_json) -> int { bool failed{}; auto rule_file_map = Base::CreateRuleFileMap(repo_config, jobs); nlohmann::json rules_file; { TaskSystem ts{jobs}; rule_file_map.ConsumeAfterKeysReady( &ts, {rule_name.ToModule()}, [&rules_file](auto values) { rules_file = *values[0]; }, [&failed](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While searching for rule definition:\n{}", msg); failed = failed or fatal; }); } if (failed) { return kExitAnalysisFailure; } auto ruledesc_it = rules_file.find(rule_name.GetNamedTarget().name); if (ruledesc_it == rules_file.end()) { Logger::Log(LogLevel::Error, "Rule definition of {} is missing", rule_name.ToString()); return kExitAnalysisFailure; } if (print_json) { PrintRuleAsOrderedJson(*ruledesc_it, rule_name.ToJson()); return kExitSuccess; } PrettyPrintRule(*ruledesc_it, rule_name, repo_config); return kExitSuccess; } auto DescribeTarget(BuildMaps::Target::ConfiguredTarget const& id, gsl::not_null const& repo_config, std::optional const& serve, ApiBundle const& apis, std::size_t jobs, bool print_json) -> int { // check if target root is absent if (repo_config->TargetRoot(id.target.ToModule().repository)->IsAbsent()) { // check that we have a serve endpoint configured if (not serve) { Logger::Log(LogLevel::Error, fmt::format("Root for target {} is absent but no serve " "endpoint was configured. Please provide " "--remote-serve-address and retry.", id.target.ToJson().dump())); return kExitBuildEnvironment; } // check that just serve and the client use same remote execution // endpoint; it might make sense in the future to remove or avoid this // check, e.g., if remote endpoints are behind proxies. if (not serve->CheckServeRemoteExecution()) { Logger::Log(LogLevel::Error, "Inconsistent remote execution endpoint and serve " "endpoint configuration detected."); return kExitBuildEnvironment; } // ask serve endpoint to provide the description auto const& repo_name = id.target.ToModule().repository; auto target_root_id = repo_config->TargetRoot(repo_name)->GetAbsentTreeId(); if (not target_root_id) { Logger::Log( LogLevel::Error, "Failed to get the target root id for repository \"{}\"", repo_name); return kExitBuildEnvironment; } if (auto dgst = serve->ServeTargetDescription( *target_root_id, *(repo_config->TargetFileName(repo_name)), id.target.GetNamedTarget().name)) { // if we're only asked to provide rule description as JSON, as this // is an export target, we don't need the blob and can directly // provide the user the information if (print_json) { std::cout << nlohmann::json({{"type", "export"}}).dump(2) << std::endl; return kExitSuccess; } auto const& desc_info = Artifact::ObjectInfo{.digest = *dgst, .type = ObjectType::File}; if (not apis.local->IsAvailable(*dgst)) { if (not apis.remote->RetrieveToCas({desc_info}, *apis.local)) { Logger::Log(LogLevel::Error, "Failed to retrieve blob {} from remote CAS", desc_info.ToString()); return kExitBuildEnvironment; } } auto const desc_str = apis.local->RetrieveToMemory(desc_info); if (not desc_str) { Logger::Log(LogLevel::Error, "Could not load in memory blob {}", desc_info.ToString()); return kExitBuildEnvironment; } // parse blob into JSON object nlohmann::json desc; try { desc = nlohmann::json::parse(*desc_str); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Parsing served target description failed with:\n{}", ex.what()); return kExitBuildEnvironment; } // serve endpoint already checked that this target is of // "type": "export", so we can just print the description std::cout << id.ToString() << " is defined by built-in rule \"export\"." << std::endl; auto doc = desc.find("doc"); if (doc != desc.end()) { PrintDoc(*doc, " | "); } auto config_doc = nlohmann::json::object(); auto config_doc_it = desc.find("config_doc"); if (config_doc_it != desc.end() and config_doc_it->is_object()) { config_doc = *config_doc_it; } auto flexible_config = desc.find("flexible_config"); if (flexible_config != desc.end() and (not flexible_config->empty())) { std::cout << " Flexible configuration variables\n"; PrintFields(*flexible_config, config_doc, " - ", " | "); } return kExitSuccess; } // report failure to serve description Logger::Log(LogLevel::Error, "Serve endpoint could not provide description of target {} " "with absent root.", id.target.ToJson().dump()); return kExitBuildEnvironment; } // process with a present target root auto targets_file_map = Base::CreateTargetsFileMap(repo_config, jobs); nlohmann::json targets_file{}; bool failed{false}; { TaskSystem ts{jobs}; targets_file_map.ConsumeAfterKeysReady( &ts, {id.target.ToModule()}, [&targets_file](auto values) { targets_file = *values[0]; }, [&failed](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While searching for target description:\n{}", msg); failed = failed or fatal; }); } if (failed) { return kExitAnalysisFailure; } auto desc_it = targets_file.find(id.target.GetNamedTarget().name); if (desc_it == targets_file.end()) { std::cout << id.ToString() << " is implicitly a source file." << std::endl; return kExitSuccess; } nlohmann::json desc = *desc_it; auto rule_it = desc.find("type"); if (rule_it == desc.end()) { Logger::Log(LogLevel::Error, "{} is a target without specified type.", id.ToString()); return kExitAnalysisFailure; } if (BuildMaps::Target::IsBuiltInRule(*rule_it)) { if (print_json) { // For built-in rules, we have no user-defined description to // provide other than informing the user that it is a built-in rule. std::cout << nlohmann::json({{"type", *rule_it}}).dump(2) << std::endl; return kExitSuccess; } std::cout << id.ToString() << " is defined by built-in rule " << rule_it->dump() << "." << std::endl; if (*rule_it == "export") { // export targets may have doc fields of their own. auto doc = desc.find("doc"); if (doc != desc.end()) { PrintDoc(*doc, " | "); } auto config_doc = nlohmann::json::object(); auto config_doc_it = desc.find("config_doc"); if (config_doc_it != desc.end() and config_doc_it->is_object()) { config_doc = *config_doc_it; } auto flexible_config = desc.find("flexible_config"); if (flexible_config != desc.end() and (not flexible_config->empty())) { std::cout << " Flexible configuration variables\n"; PrintFields(*flexible_config, config_doc, " - ", " | "); } } else if (*rule_it == "configure") { auto target = desc.find("target"); auto doc = desc.find("doc"); if (doc != desc.end()) { PrintDoc(*doc, " | "); } if (target != desc.end()) { std::cout << "The target to be configured is defined as " << target->dump() << "." << std::endl; } } return kExitSuccess; } auto rule_name = BuildMaps::Base::ParseEntityNameFromJson( *rule_it, id.target, repo_config, [&rule_it, &id](std::string const& parse_err) { Logger::Log(LogLevel::Error, "Parsing rule name {} for target {} failed with:\n{}.", rule_it->dump(), id.ToString(), parse_err); }); if (not rule_name) { return kExitAnalysisFailure; } if (not print_json) { std::cout << id.ToString() << " is defined by user-defined rule " << rule_name->ToString() << ".\n\n"; } return DescribeUserDefinedRule(*rule_name, repo_config, jobs, print_json); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/describe.hpp000066400000000000000000000031261516554100600256220ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_DESCRIBE_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_DESCRIBE_HPP #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" [[nodiscard]] auto DescribeTarget( BuildMaps::Target::ConfiguredTarget const& id, gsl::not_null const& repo_config, std::optional const& serve, ApiBundle const& apis, std::size_t jobs, bool print_json) -> int; [[nodiscard]] auto DescribeUserDefinedRule( BuildMaps::Base::EntityName const& rule_name, gsl::not_null const& repo_config, std::size_t jobs, bool print_json) -> int; #endif // INCLUDED_SRC_BUILDTOOL_MAIN_DESCRIBE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/diagnose.cpp000066400000000000000000000327531516554100600256360ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/diagnose.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/json.hpp" namespace { namespace Base = BuildMaps::Base; namespace Target = BuildMaps::Target; [[nodiscard]] auto ResultToJson(TargetResult const& result) -> nlohmann::json { return nlohmann::ordered_json{ {"artifacts", result.artifact_stage->ToJson( Expression::JsonMode::SerializeAllButNodes)}, {"runfiles", result.runfiles->ToJson(Expression::JsonMode::SerializeAllButNodes)}, {"provides", result.provides->ToJson(Expression::JsonMode::SerializeAllButNodes)}}; } [[nodiscard]] auto TargetActionsToJson(AnalysedTargetPtr const& target) -> nlohmann::json { auto actions = nlohmann::json::array(); std::for_each(target->Actions().begin(), target->Actions().end(), [&actions](auto const& action) { actions.push_back(action->ToJson()); }); return actions; } [[nodiscard]] auto TreesToJson(AnalysedTargetPtr const& target) -> nlohmann::json { auto trees = nlohmann::json::object(); std::for_each( target->Trees().begin(), target->Trees().end(), [&trees](auto const& tree) { trees[tree->Id()] = tree->ToJson(); }); return trees; } void DumpActions(std::string const& file_path, AnalysisResult const& result) { auto const dump_string = IndentListsOnlyUntilDepth(TargetActionsToJson(result.target), 2, 1); if (file_path == "-") { Logger::Log( LogLevel::Info, "Actions for target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping actions for target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpBlobs(std::string const& file_path, AnalysisResult const& result) { auto blobs = nlohmann::json::array(); for (auto const& s : result.target->Blobs()) { blobs.push_back(s); } auto const dump_string = blobs.dump(2); if (file_path == "-") { Logger::Log( LogLevel::Info, "Blobs for target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping blobs for target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpVars(std::string const& file_path, AnalysisResult const& result) { auto vars = std::vector{}; vars.reserve(result.target->Vars().size()); for (auto const& x : result.target->Vars()) { vars.push_back(x); } std::sort(vars.begin(), vars.end()); auto const dump_string = nlohmann::json(vars).dump(); if (file_path == "-") { Logger::Log( LogLevel::Info, "Variables for target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping variables for target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpTrees(std::string const& file_path, AnalysisResult const& result) { auto const dump_string = TreesToJson(result.target).dump(2); if (file_path == "-") { Logger::Log( LogLevel::Info, "Trees for target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping trees for target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpProvides(std::string const& file_path, AnalysisResult const& result) { auto const dump_string = result.target->Result() .provides->ToJson(Expression::JsonMode::SerializeAllButNodes) .dump(2); if (file_path == "-") { Logger::Log(LogLevel::Info, "Provides map for target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping provides map for target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpTargets(std::string const& file_path, std::vector const& target_ids, std::string const& target_qualifier = "") { auto repo_map = nlohmann::json::object(); auto conf_list = [&repo_map](Base::EntityName const& ref) -> nlohmann::json& { if (ref.IsAnonymousTarget()) { auto const& anon = ref.GetAnonymousTarget(); auto& anon_map = repo_map[Base::EntityName::kAnonymousMarker]; auto& rule_map = anon_map[anon.rule_map.ToIdentifier()]; return rule_map[anon.target_node.ToIdentifier()]; } auto const& named = ref.GetNamedTarget(); auto& location_map = repo_map[Base::EntityName::kLocationMarker]; auto& module_map = location_map[named.repository]; auto& target_map = module_map[named.module]; return target_map[named.name]; }; std::for_each( target_ids.begin(), target_ids.end(), [&conf_list](auto const& id) { if ((not id.target.IsNamedTarget()) or id.target.GetNamedTarget().reference_t == BuildMaps::Base::ReferenceType::kTarget) { conf_list(id.target).push_back(id.config.ToJson()); } }); auto const dump_string = IndentListsOnlyUntilDepth(repo_map, 2); if (file_path == "-") { Logger::Log( LogLevel::Info, "List of analysed {}targets:", target_qualifier); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping list of analysed {}targets to file '{}'.", target_qualifier, file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } auto DumpExpressionToMap(gsl::not_null const& map, ExpressionPtr const& expr) -> bool { auto const& id = expr->ToIdentifier(); if (not map->contains(id)) { (*map)[id] = expr->ToJson(); return true; } return false; } void DumpNodesInExpressionToMap(gsl::not_null const& map, ExpressionPtr const& expr) { if (expr->IsNode()) { if (DumpExpressionToMap(map, expr)) { auto const& node = expr->Node(); if (node.IsAbstract()) { DumpNodesInExpressionToMap(map, node.GetAbstract().target_fields); } else if (node.IsValue()) { DumpNodesInExpressionToMap(map, node.GetValue()); } } } else if (expr->IsList()) { for (auto const& entry : expr->List()) { DumpNodesInExpressionToMap(map, entry); } } else if (expr->IsMap()) { for (auto const& [_, value] : expr->Map()) { DumpNodesInExpressionToMap(map, value); } } else if (expr->IsResult()) { DumpNodesInExpressionToMap(map, expr->Result().provides); } } void DumpAnonymous(std::string const& file_path, std::vector const& target_ids) { auto anon_map = nlohmann::json{{"nodes", nlohmann::json::object()}, {"rule_maps", nlohmann::json::object()}}; std::for_each( target_ids.begin(), target_ids.end(), [&anon_map](auto const& id) { if (id.target.IsAnonymousTarget()) { auto const& anon_t = id.target.GetAnonymousTarget(); DumpExpressionToMap(&anon_map["rule_maps"], anon_t.rule_map); DumpNodesInExpressionToMap(&anon_map["nodes"], anon_t.target_node); } }); auto const dump_string = IndentListsOnlyUntilDepth(anon_map, 2); if (file_path == "-") { Logger::Log(LogLevel::Info, "List of anonymous target data:"); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping list of anonymous target data to file '{}'.", file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpNodes(std::string const& file_path, AnalysisResult const& result) { auto node_map = nlohmann::json::object(); DumpNodesInExpressionToMap(&node_map, result.target->Provides()); auto const dump_string = IndentListsOnlyUntilDepth(node_map, 2); if (file_path == "-") { Logger::Log( LogLevel::Info, "Target nodes of target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping target nodes of target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os(file_path); os << dump_string << std::endl; } } void DumpResult(std::string const& file_path, AnalysisResult const& result) { auto const dump_string = ResultToJson(result.target->Result()).dump(); if (file_path == "-") { Logger::Log( LogLevel::Info, "Result of target {}:", result.id.ToString()); std::cout << dump_string << std::endl; } else { Logger::Log(LogLevel::Info, "Dumping result of target {} to file '{}'.", result.id.ToString(), file_path); std::ofstream os{file_path}; os << dump_string << std::endl; } } } // namespace void DiagnoseResults(AnalysisResult const& result, DiagnosticArguments const& clargs) { Logger::Log( LogLevel::Info, "Result of{} target {}: {}", result.modified ? fmt::format(" input of action {} of", *result.modified) : "", result.id.ToString(), IndentOnlyUntilDepth( ResultToJson(result.target->Result()), 2, 2, std::unordered_map{{"/provides", 3}})); if (clargs.dump_result) { DumpResult(*clargs.dump_result, result); } if (clargs.dump_actions) { DumpActions(*clargs.dump_actions, result); } if (clargs.dump_blobs) { DumpBlobs(*clargs.dump_blobs, result); } if (clargs.dump_trees) { DumpTrees(*clargs.dump_trees, result); } if (clargs.dump_provides) { DumpProvides(*clargs.dump_provides, result); } if (clargs.dump_vars) { DumpVars(*clargs.dump_vars, result); } if (clargs.dump_targets) { DumpTargets(*clargs.dump_targets, result.result_map.ConfiguredTargets()); } if (clargs.dump_export_targets) { DumpTargets(*clargs.dump_export_targets, result.result_map.ExportTargets(), "export "); } if (clargs.dump_targets_graph) { auto graph = result.result_map.ConfiguredTargetsGraph().dump(2); Logger::Log(LogLevel::Info, "Dumping graph of configured-targets to file {}.", *clargs.dump_targets_graph); std::ofstream os(*clargs.dump_targets_graph); os << graph << std::endl; } if (clargs.dump_anonymous) { DumpAnonymous(*clargs.dump_anonymous, result.result_map.ConfiguredTargets()); } if (clargs.dump_nodes) { DumpNodes(*clargs.dump_nodes, result); } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/diagnose.hpp000066400000000000000000000017031516554100600256320ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_DIAGNOSE_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_DIAGNOSE_HPP #include "src/buildtool/common/cli.hpp" #include "src/buildtool/main/analyse.hpp" void DiagnoseResults(AnalysisResult const& result, DiagnosticArguments const& clargs); #endif // INCLUDED_SRC_BUILDTOOL_MAIN_DIAGNOSE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/exit_codes.hpp000066400000000000000000000017531516554100600261740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_EXIT_CODES_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_EXIT_CODES_HPP #include enum ExitCodes : std::uint8_t { kExitSuccess = 0, kExitFailure = 1, kExitSuccessFailedArtifacts = 2, kExitAnalysisFailure = 8, kExitBuildEnvironment = 16, kExitSyntaxError = 32 }; #endif // INCLUDED_SRC_BUILDTOOL_MAIN_EXIT_CODES_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/install_cas.cpp000066400000000000000000000146661516554100600263440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/install_cas.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/utils/subobject.hpp" #include "src/buildtool/main/archive.hpp" #endif namespace { [[nodiscard]] auto InvalidSizeString(HashFunction::Type hash_type, std::string const& size_str, std::string const& hash, bool has_remote) noexcept -> bool { // Only in compatible mode the size is checked, so an empty SHA256 hash is // needed. static auto const kEmptyHash = HashFunction{HashFunction::Type::PlainSHA256}.HashBlobData(""); return hash_type == HashFunction::Type::PlainSHA256 and // native mode is fine (size_str == "0" or size_str.empty()) and // not "0" or "" is fine kEmptyHash.HexString() != hash and // empty hash is fine has_remote; // local is fine } } // namespace [[nodiscard]] auto ObjectInfoFromLiberalString(HashFunction::Type hash_type, std::string const& s, bool has_remote) noexcept -> std::optional { std::istringstream iss(s); std::string id{}; std::string size_str{}; std::string type{"f"}; if (iss.peek() == '[') { (void)iss.get(); } std::getline(iss, id, ':'); if (not iss.eof()) { std::getline(iss, size_str, ':'); } if (not iss.eof()) { std::getline(iss, type, ']'); } if (InvalidSizeString(hash_type, size_str, id, has_remote)) { Logger::Log( LogLevel::Warning, "{} size in object-id is not supported in compatibility mode.", size_str.empty() ? "omitting the" : "zero"); } auto size = static_cast( size_str.empty() ? 0 : std::atol(size_str.c_str())); auto const& object_type = FromChar(*type.c_str()); auto digest = ArtifactDigestFactory::Create( hash_type, id, size, IsTreeObject(object_type)); if (not digest) { return std::nullopt; } return Artifact::ObjectInfo{.digest = *std::move(digest), .type = object_type}; } #ifndef BOOTSTRAP_BUILD_TOOL auto FetchAndInstallArtifacts(ApiBundle const& apis, FetchArguments const& clargs, RemoteContext const& remote_context) -> bool { auto object_info = ObjectInfoFromLiberalString( apis.remote->GetHashType(), clargs.object_id, remote_context.exec_config->remote_address.has_value()); if (not object_info) { return false; } if (clargs.remember) { if (not apis.remote->ParallelRetrieveToCas( {*object_info}, *apis.local, 1, true)) { Logger::Log(LogLevel::Warning, "Failed to copy artifact {} to local CAS", object_info->ToString()); } } if (clargs.sub_path) { auto new_object_info = RetrieveSubPathId(*object_info, apis, *clargs.sub_path); if (new_object_info) { object_info = new_object_info; } else { return false; } } std::optional out{}; if (clargs.output_path) { // Compute output location and create parent directories auto output_path = (*clargs.output_path / "").parent_path(); if (FileSystemManager::IsDirectory(output_path)) { output_path /= object_info->digest.hash(); } if (not FileSystemManager::CreateDirectory(output_path.parent_path())) { Logger::Log(LogLevel::Error, "failed to create parent directory {}.", output_path.parent_path().string()); return false; } out = output_path; } if (clargs.archive) { if (object_info->type != ObjectType::Tree) { Logger::Log(LogLevel::Error, "Archive requested on non-tree {}", object_info->ToString()); return false; } return GenerateArchive(*apis.remote, *object_info, out); } if (out) { if (not apis.remote->RetrieveToPaths( {*object_info}, {*out}, &*apis.local)) { Logger::Log(LogLevel::Error, "failed to retrieve artifact."); return false; } Logger::Log(LogLevel::Info, "artifact {} was installed to {}", object_info->ToString(), out->string()); } else { // dump to stdout if (not apis.remote->RetrieveToFds({*object_info}, {dup(fileno(stdout))}, clargs.raw_tree, &*apis.local)) { Logger::Log(LogLevel::Error, "failed to dump artifact."); return false; } } return true; } #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/install_cas.hpp000066400000000000000000000032411516554100600263340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_INSTALL_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_INSTALL_CAS_HPP #include #include #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/crypto/hash_function.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/common/cli.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #endif /// \note Method is public for use also in tests. [[nodiscard]] auto ObjectInfoFromLiberalString(HashFunction::Type hash_type, std::string const& s, bool has_remote) noexcept -> std::optional; #ifndef BOOTSTRAP_BUILD_TOOL [[nodiscard]] auto FetchAndInstallArtifacts(ApiBundle const& apis, FetchArguments const& clargs, RemoteContext const& remote_context) -> bool; #endif #endif // INCLUDED_SRC_BUILDTOOL_MAIN_INSTALL_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/main.cpp000066400000000000000000001524651516554100600247740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" #include "src/buildtool/logging/log_sink_file.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/add_to_cas.hpp" #include "src/buildtool/main/analyse.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/buildtool/main/cli.hpp" #include "src/buildtool/main/constants.hpp" #include "src/buildtool/main/diagnose.hpp" #include "src/buildtool/main/exit_codes.hpp" #include "src/buildtool/main/install_cas.hpp" #include "src/buildtool/main/version.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/profile/profile.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/file_chunker.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/json.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "fmt/core.h" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/computed_roots/evaluate.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/execution_service/server_implementation.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/main/describe.hpp" #include "src/buildtool/main/retry.hpp" #include "src/buildtool/main/serve.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/serve_service/serve_server_implementation.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #endif // BOOTSTRAP_BUILD_TOOL namespace { namespace Base = BuildMaps::Base; namespace Target = BuildMaps::Target; void SetupDefaultLogging() { LogConfig::SetLogLimit(kDefaultLogLevel); LogConfig::SetSinks({LogSinkCmdLine::CreateFactory()}); } void SetupLogging(LogArguments const& clargs) { LogConfig::SetLogLimit(clargs.log_limit); LogConfig::SetSinks({LogSinkCmdLine::CreateFactory( not clargs.plain_log, clargs.restrict_stderr_log_limit)}); for (auto const& log_file : clargs.log_files) { LogConfig::AddSink(LogSinkFile::CreateFactory( log_file, clargs.log_append ? LogSinkFile::Mode::Append : LogSinkFile::Mode::Overwrite)); } } [[nodiscard]] auto CreateStorageConfig( EndpointArguments const& eargs, HashFunction::Type hash_type, std::optional const& remote_address = std::nullopt, ExecutionProperties const& remote_platform_properties = {}, std::vector const& remote_dispatch = {}) noexcept -> std::optional { StorageConfig::Builder builder; if (eargs.local_root.has_value()) { builder.SetBuildRoot(*eargs.local_root); } auto backend_description = BackendDescription::Describe( remote_address, remote_platform_properties, remote_dispatch); if (not backend_description) { Logger::Log(LogLevel::Error, std::move(backend_description).error()); return std::nullopt; } auto config = builder.SetHashType(hash_type) .SetBackendDescription(std::move(backend_description).value()) .Build(); if (not config) { Logger::Log(LogLevel::Error, std::move(config).error()); return std::nullopt; } return *std::move(config); } #ifndef BOOTSTRAP_BUILD_TOOL [[nodiscard]] auto CreateLocalExecutionConfig( BuildArguments const& bargs) noexcept -> std::optional { LocalExecutionConfig::Builder builder; if (bargs.local_launcher.has_value()) { builder.SetLauncher(*bargs.local_launcher); } auto config = builder.Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } [[nodiscard]] auto CreateRemoteExecutionConfig(EndpointArguments const& eargs, RebuildArguments const& rargs) -> std::optional { RemoteExecutionConfig::Builder builder; builder.SetRemoteAddress(eargs.remote_execution_address) .SetRemoteInstanceName(eargs.remote_instance_name) .SetRemoteExecutionDispatch(eargs.remote_execution_dispatch_file) .SetPlatformProperties(eargs.platform_properties) .SetCacheAddress(rargs.cache_endpoint); auto config = builder.Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } [[nodiscard]] auto CreateServeConfig(ServeArguments const& srvargs, CommonArguments const& cargs, BuildArguments const& bargs, TCArguments const& tc) noexcept -> std::optional { RemoteServeConfig::Builder builder; builder.SetRemoteAddress(srvargs.remote_serve_address) .SetClientExecutionAddress(srvargs.client_remote_address) .SetKnownRepositories(srvargs.repositories) .SetJobs(cargs.jobs) .SetActionTimeout(bargs.timeout) .SetTCStrategy(tc.target_cache_write_strategy); if (bargs.build_jobs > 0) { builder.SetBuildJobs(bargs.build_jobs); } auto config = builder.Build(); if (config) { if (config->tc_strategy == TargetCacheWriteStrategy::Disable) { Logger::Log( LogLevel::Info, "Target-level cache writing of serve service is disabled."); } return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } [[nodiscard]] auto CreateAuthConfig( CommonAuthArguments const& authargs, ClientAuthArguments const& client_authargs, ServerAuthArguments const& server_authargs) noexcept -> std::optional { Auth::TLS::Builder tls_builder; tls_builder.SetCACertificate(authargs.tls_ca_cert) .SetClientCertificate(client_authargs.tls_client_cert) .SetClientKey(client_authargs.tls_client_key) .SetServerCertificate(server_authargs.tls_server_cert) .SetServerKey(server_authargs.tls_server_key); // create auth config (including validation) auto result = tls_builder.Build(); if (result) { if (*result) { // correctly configured TLS/SSL certification return *std::move(*result); } Logger::Log(LogLevel::Error, result->error()); return std::nullopt; } // no TLS/SSL configuration was given, and we currently support no other // certification method, so return an empty config (no certification) return Auth{}; } void SetupFileChunker() { FileChunker::Initialize(); } #endif // BOOTSTRAP_BUILD_TOOL // returns path relative to `root`. [[nodiscard]] auto FindRoot(std::filesystem::path const& subdir, FileRoot const& root, std::vector const& markers) -> std::optional { Expects(subdir.is_relative()); auto current = subdir; while (true) { for (auto const& marker : markers) { if (root.Exists(current / marker)) { return current; } } if (current.empty()) { break; } current = current.parent_path(); } return std::nullopt; } [[nodiscard]] auto ReadConfiguration(AnalysisArguments const& clargs) noexcept -> Configuration { Configuration config{}; if (not clargs.config_file.empty()) { if (not FileSystemManager::Exists(clargs.config_file)) { Logger::Log(LogLevel::Error, "Config file {} does not exist.", clargs.config_file.string()); std::exit(kExitFailure); } try { std::ifstream fs(clargs.config_file); auto map = Expression::FromJson(nlohmann::json::parse(fs)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "Config file {} does not contain a map.", clargs.config_file.string()); std::exit(kExitFailure); } config = Configuration{map}; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing config file {} failed with error:\n{}", clargs.config_file.string(), e.what()); std::exit(kExitFailure); } } for (auto const& s : clargs.defines) { try { auto map = Expression::FromJson(nlohmann::json::parse(s)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "Defines entry {} does not contain a map.", nlohmann::json(s).dump()); std::exit(kExitFailure); } config = config.Update(map); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing defines entry {} failed with error:\n{}", nlohmann::json(s).dump(), e.what()); std::exit(kExitFailure); } } return config; } [[nodiscard]] auto DetermineCurrentModule( std::filesystem::path const& workspace_root, FileRoot const& target_root, std::string const& target_file_name) -> std::string { std::filesystem::path cwd{}; try { cwd = std::filesystem::current_path(); } catch (std::exception const& e) { Logger::Log(LogLevel::Warning, "Cannot determine current working directory ({}), assuming " "top-level module is requested", e.what()); return "."; } auto subdir = std::filesystem::proximate(cwd, workspace_root); if (subdir.is_relative() and (*subdir.begin() != "..")) { // cwd is subdir of workspace_root if (auto root_dir = FindRoot(subdir, target_root, {target_file_name})) { return root_dir->string(); } } return "."; } [[nodiscard]] auto ReadConfiguredTarget( std::string const& main_repo, std::optional const& main_ws_root, gsl::not_null const& repo_config, AnalysisArguments const& clargs) -> Target::ConfiguredTarget { auto const* target_root = repo_config->TargetRoot(main_repo); if (target_root == nullptr) { Logger::Log(LogLevel::Error, "Cannot obtain target root for main repo {}.", main_repo); std::exit(kExitFailure); } auto current_module = std::string{"."}; std::string target_file_name = *repo_config->TargetFileName(main_repo); if (main_ws_root) { // module detection only works if main workspace is on the file system current_module = DetermineCurrentModule( *main_ws_root, *target_root, target_file_name); } auto config = ReadConfiguration(clargs); if (clargs.target) { auto entity = Base::ParseEntityNameFromJson( *clargs.target, Base::EntityName{Base::NamedTarget{main_repo, current_module, ""}}, repo_config, [&clargs](std::string const& parse_err) { Logger::Log(LogLevel::Error, "Parsing target name {} failed with:\n{}", clargs.target->dump(), parse_err); }); if (not entity) { std::exit(kExitFailure); } return Target::ConfiguredTarget{.target = std::move(*entity), .config = std::move(config)}; } #ifndef BOOTSTRAP_BUILD_TOOL if (target_root->IsAbsent()) { // since the user has not specified the target to build, we use the most // reasonable default value: // // module -> "." (i.e., current module) // target -> "" (i.e., firstmost lexicographical target name) auto target = nlohmann::json::parse(R"([".",""])"); Logger::Log(LogLevel::Debug, "Detected absent target root for repo {} and no target was " "given. Assuming default target {}", main_repo, target.dump()); auto entity = Base::ParseEntityNameFromJson( target, Base::EntityName{Base::NamedTarget{main_repo, current_module, ""}}, repo_config, [&target](std::string const& parse_err) { Logger::Log(LogLevel::Error, "Parsing target name {} failed with:\n{}", target.dump(), parse_err); }); if (not entity) { std::exit(kExitFailure); } return Target::ConfiguredTarget{.target = std::move(*entity), .config = std::move(config)}; } #endif auto const target_file = (std::filesystem::path{current_module} / target_file_name).string(); if (not target_root->IsFile(target_file)) { Logger::Log(LogLevel::Error, "Expected file at {}.", target_file); std::exit(kExitFailure); } auto file_content = target_root->ReadContent(target_file); if (not file_content) { Logger::Log(LogLevel::Error, "Cannot read file {}.", target_file); std::exit(kExitFailure); } auto json = nlohmann::json(); try { json = nlohmann::json::parse(*file_content); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "While searching for the default target in {}:\n" "Failed to parse json with error {}", target_file, e.what()); std::exit(kExitFailure); } if (not json.is_object()) { Logger::Log( LogLevel::Error, "Invalid content in target file {}.", target_file); std::exit(kExitFailure); } if (json.empty()) { Logger::Log(LogLevel::Error, "Missing target descriptions in file {}.", target_file); std::exit(kExitFailure); } return Target::ConfiguredTarget{ .target = Base::EntityName{Base::NamedTarget{ main_repo, current_module, json.begin().key()}}, .config = std::move(config)}; } [[nodiscard]] auto DetermineWorkspaceRootByLookingForMarkers() noexcept -> std::filesystem::path { std::filesystem::path cwd{}; try { cwd = std::filesystem::current_path(); } catch (std::exception const& e) { Logger::Log(LogLevel::Warning, "Failed to determine current working directory ({})", e.what()); Logger::Log(LogLevel::Error, "Could not determine workspace root."); std::exit(kExitFailure); } auto root = cwd.root_path(); cwd = std::filesystem::relative(cwd, root); auto root_dir = FindRoot(cwd, FileRoot{root}, kRootMarkers); if (not root_dir) { Logger::Log(LogLevel::Error, "Could not determine workspace root."); std::exit(kExitFailure); } return root / *root_dir; } // Set all roots and name mappings from the command-line arguments and // return the name of the main repository and main workspace path if local. auto DetermineRoots(gsl::not_null const& storage_config, gsl::not_null const& repository_config, CommonArguments const& cargs, AnalysisArguments const& aargs) -> std::pair> { std::optional main_ws_root; auto repo_config = nlohmann::json::object(); if (cargs.repository_config) { try { std::ifstream fs(*cargs.repository_config); repo_config = nlohmann::json::parse(fs); if (not repo_config.is_object()) { Logger::Log( LogLevel::Error, "Repository configuration file {} does not contain a map.", (*cargs.repository_config).string()); std::exit(kExitFailure); } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing repository configuration file {} failed with " "error:\n{}", (*cargs.repository_config).string(), e.what()); std::exit(kExitFailure); } } std::string main_repo; if (cargs.main) { main_repo = *cargs.main; } else if (auto main_it = repo_config.find("main"); main_it != repo_config.end()) { if (not main_it->is_string()) { Logger::Log(LogLevel::Error, "Repository config: main has to be a string"); std::exit(kExitFailure); } main_repo = *main_it; } else if (auto repos_it = repo_config.find("repositories"); repos_it != repo_config.end() and not repos_it->empty()) { // take lexicographical first key as main (nlohmann::json is sorted) main_repo = repos_it->begin().key(); } auto repos = nlohmann::json::object(); auto repos_it = repo_config.find("repositories"); if (repos_it != repo_config.end()) { if (not repos_it->is_object()) { Logger::Log(LogLevel::Error, "Repository config: repositories has to be a map"); std::exit(kExitFailure); } repos = *repos_it; } if (not repos.contains(main_repo)) { repos[main_repo] = nlohmann::json::object(); } for (auto const& [repo, desc] : repos.items()) { std::optional ws_root{}; bool const is_main_repo{repo == main_repo}; auto it_ws = desc.find("workspace_root"); if (it_ws != desc.end()) { if (auto parsed_root = FileRoot::ParseRoot( storage_config, repo, "workspace_root", *it_ws)) { auto result = *std::move(parsed_root); ws_root = std::move(result.first); if (is_main_repo and result.second.has_value()) { main_ws_root = std::move(result.second); } } else { Logger::Log(LogLevel::Error, parsed_root.error()); std::exit(kExitFailure); } } if (is_main_repo) { // command line argument always overwrites what is eventually found // in the config file if (cargs.workspace_root) { main_ws_root = *cargs.workspace_root; } else if (not ws_root) { main_ws_root = DetermineWorkspaceRootByLookingForMarkers(); } if (main_ws_root.has_value()) { ws_root = FileRoot{*main_ws_root}; } } if (not ws_root) { Logger::Log(LogLevel::Error, "Unknown workspace root for repository {}", repo); std::exit(kExitFailure); } auto info = RepositoryConfig::RepositoryInfo{std::move(*ws_root)}; auto parse_keyword_root = [&desc = desc, &repo = repo, is_main_repo, storage_config]( FileRoot* keyword_root, std::string const& keyword, auto const& keyword_carg) { auto it = desc.find(keyword); if (it != desc.end()) { if (auto parsed_root = FileRoot::ParseRoot( storage_config, repo, keyword, *it)) { (*keyword_root) = std::move(parsed_root)->first; } else { Logger::Log(LogLevel::Error, parsed_root.error()); std::exit(kExitFailure); } } if (is_main_repo and keyword_carg) { *keyword_root = FileRoot{*keyword_carg}; } }; info.target_root = info.workspace_root; parse_keyword_root(&info.target_root, "target_root", aargs.target_root); info.rule_root = info.target_root; parse_keyword_root(&info.rule_root, "rule_root", aargs.rule_root); info.expression_root = info.rule_root; parse_keyword_root( &info.expression_root, "expression_root", aargs.expression_root); auto it_bindings = desc.find("bindings"); if (it_bindings != desc.end()) { if (not it_bindings->is_object()) { Logger::Log( LogLevel::Error, "bindings has to be a string-string map, but found {}", it_bindings->dump()); std::exit(kExitFailure); } for (auto const& [local_name, global_name] : it_bindings->items()) { if (not repos.contains(global_name)) { Logger::Log(LogLevel::Error, "Binding {} for {} in {} does not refer to a " "defined repository.", global_name, local_name, repo); std::exit(kExitFailure); } info.name_mapping[local_name] = global_name; } } auto parse_keyword_file_name = [&desc = desc, is_main_repo]( std::string* keyword_file_name, std::string const& keyword, auto const& keyword_carg) { auto it = desc.find(keyword); if (it != desc.end()) { *keyword_file_name = *it; } if (is_main_repo and keyword_carg) { *keyword_file_name = *keyword_carg; } }; parse_keyword_file_name( &info.target_file_name, "target_file_name", aargs.target_file_name); parse_keyword_file_name( &info.rule_file_name, "rule_file_name", aargs.rule_file_name); parse_keyword_file_name(&info.expression_file_name, "expression_file_name", aargs.expression_file_name); repository_config->SetInfo(repo, std::move(info)); } return {main_repo, main_ws_root}; } void ReportTaintedness(const AnalysisResult& result) { if (result.target->Tainted().empty()) { // Never report untainted targets return; } // To ensure proper quoting, go through json. nlohmann::json tainted{}; for (auto const& s : result.target->Tainted()) { tainted.push_back(s); } Logger::Log(LogLevel::Info, "Target tainted {}.", tainted.dump()); } auto DetermineNonExplicitTarget( std::string const& main_repo, std::optional const& main_ws_root, gsl::not_null const& repo_config, AnalysisArguments const& clargs) -> std::optional { auto id = ReadConfiguredTarget(main_repo, main_ws_root, repo_config, clargs); switch (id.target.GetNamedTarget().reference_t) { case Base::ReferenceType::kFile: std::cout << id.ToString() << " is a source file." << std::endl; return std::nullopt; case Base::ReferenceType::kTree: std::cout << id.ToString() << " is a tree." << std::endl; return std::nullopt; case Base::ReferenceType::kGlob: std::cout << id.ToString() << " is a glob." << std::endl; return std::nullopt; case Base::ReferenceType::kSymlink: std::cout << id.ToString() << " is a symlink." << std::endl; return std::nullopt; case Base::ReferenceType::kTarget: return id; } return std::nullopt; // make gcc happy } void DumpArtifactsToBuild( std::map const& artifacts, std::map const& runfiles, const std::filesystem::path& file_path) { nlohmann::json to_build{}; for (auto const& [path, artifact] : runfiles) { to_build[path] = artifact.ToJson(); } for (auto const& [path, artifact] : artifacts) { to_build[path] = artifact.ToJson(); } auto const dump_string = IndentListsOnlyUntilDepth(to_build, 2, 1); std::ofstream os(file_path); os << dump_string << std::endl; } } // namespace auto main(int argc, char* argv[]) -> int { std::unique_ptr profile; SetupDefaultLogging(); try { auto arguments = ParseCommandLineArguments(argc, argv); if (arguments.cmd == SubCommand::kVersion) { std::cout << version() << std::endl; return kExitSuccess; } // just serve configures all from its config file, so parse that before // doing further setup steps #ifndef BOOTSTRAP_BUILD_TOOL if (arguments.cmd == SubCommand::kServe) { ReadJustServeConfig(&arguments); } #endif // BOOTSTRAP_BUILD_TOOL SetupLogging(arguments.log); if (arguments.analysis.expression_log_limit) { Evaluator::SetExpressionLogLimit( *arguments.analysis.expression_log_limit); } // global repository configuration RepositoryConfig repo_config{}; #ifndef BOOTSTRAP_BUILD_TOOL /** * The current implementation of libgit2 uses pthread_key_t incorrectly * on POSIX systems to handle thread-specific data, which requires us to * explicitly make sure the main thread is the first one to call * git_libgit2_init. Future versions of libgit2 will hopefully fix this. */ GitContext::Create(); SetupFileChunker(); if (arguments.cmd == SubCommand::kGc) { // Set up storage for GC, as we have all the config args we need. auto const storage_config = CreateStorageConfig( arguments.endpoint, arguments.protocol.hash_type); if (not storage_config) { return kExitBuildEnvironment; } if (GarbageCollector::TriggerGarbageCollection( *storage_config, arguments.gc.no_rotate, arguments.gc.all)) { return kExitSuccess; } return kExitBuildEnvironment; } auto local_exec_config = CreateLocalExecutionConfig(arguments.build); if (not local_exec_config) { return kExitBuildEnvironment; } auto auth_config = CreateAuthConfig(arguments.auth, arguments.cauth, arguments.sauth); if (not auth_config) { return kExitBuildEnvironment; } if (arguments.cmd == SubCommand::kExecute) { auto execution_server = ServerImpl::Create(arguments.service.interface, arguments.service.port, arguments.service.info_file, arguments.service.pid_file); if (execution_server) { RetryConfig retry_config{}; // default is enough, as remote is not used // Use default remote configuration. RemoteExecutionConfig remote_exec_config{}; // Set up storage for local execution. auto const storage_config = CreateStorageConfig( arguments.endpoint, arguments.protocol.hash_type); if (not storage_config) { return kExitBuildEnvironment; } auto const storage = Storage::Create(&*storage_config); // Add the empty blob to CAS, as some build tools assume that // the empty blob can always be referred to without uploading // it. auto store_empty_blob_result = storage.CAS().StoreBlob(std::string{}); if (not store_empty_blob_result) { Logger::Log(LogLevel::Warning, "Failed to store empty blob in CAS."); } // pack the local context instances to be passed as needed LocalContext const local_context{ .exec_config = &*local_exec_config, .storage_config = &*storage_config, .storage = &storage}; // pack the remote context instances to be passed as needed RemoteContext const remote_context{ .auth = &*auth_config, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const exec_apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); return execution_server->Run( &local_context, &remote_context, dynamic_cast(&*exec_apis.local), arguments.service.op_exponent) ? kExitSuccess : kExitFailure; } return kExitBuildEnvironment; } auto serve_config = CreateServeConfig( arguments.serve, arguments.common, arguments.build, arguments.tc); if (not serve_config) { return kExitBuildEnvironment; } // Set up the retry arguments, needed only for the client-side logic of // remote execution, i.e., just serve and the regular just client. auto retry_config = CreateRetryConfig(arguments.retry); if (not retry_config) { return kExitBuildEnvironment; } if (arguments.cmd == SubCommand::kServe) { std::mutex lock{}; auto serve_server = ServeServerImpl::Create(arguments.service.interface, arguments.service.port, arguments.service.info_file, arguments.service.pid_file, &lock); if (serve_server) { // Set up remote execution config. auto remote_exec_config = CreateRemoteExecutionConfig( arguments.endpoint, arguments.rebuild); if (not remote_exec_config) { return kExitBuildEnvironment; } // Set up storage for serve operation. auto const storage_config = CreateStorageConfig(arguments.endpoint, arguments.protocol.hash_type, remote_exec_config->remote_address, remote_exec_config->platform_properties, remote_exec_config->dispatch); if (not storage_config) { return kExitBuildEnvironment; } auto const storage = Storage::Create(&*storage_config); // pack the local context instances to be passed as needed LocalContext const local_context{ .exec_config = &*local_exec_config, .storage_config = &*storage_config, .storage = &storage}; // pack the remote context instances to be passed as needed RemoteContext const remote_context{ .auth = &*auth_config, .retry_config = &*retry_config, .exec_config = &*remote_exec_config}; auto const serve_apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &serve_apis); bool with_execute = not remote_exec_config->remote_address.has_value(); // Operation cache only relevant for just execute auto const op_exponent = with_execute ? arguments.service.op_exponent : std::nullopt; return serve_server->Run(*serve_config, &local_context, &remote_context, serve, serve_apis, op_exponent, with_execute) ? kExitSuccess : kExitFailure; } return kExitBuildEnvironment; } // Setup profile logging, if requested if (arguments.analysis.profile) { profile = std::make_unique(*arguments.analysis.profile, arguments); } // If no execution endpoint was given, the client should default to the // serve endpoint, if given. if (not arguments.endpoint.remote_execution_address.has_value() and arguments.serve.remote_serve_address.has_value()) { // replace the remote execution address arguments.endpoint.remote_execution_address = *arguments.serve.remote_serve_address; // Inform user of the change Logger::Log(LogLevel::Info, "Using '{}' as the remote execution endpoint.", *arguments.serve.remote_serve_address); } // Set up remote execution config. auto remote_exec_config = CreateRemoteExecutionConfig(arguments.endpoint, arguments.rebuild); if (not remote_exec_config) { if (profile != nullptr) { profile->Write(kExitBuildEnvironment); } return kExitBuildEnvironment; } if (profile != nullptr) { profile->SetRemoteExecutionConfig(*remote_exec_config); } // Set up storage for client-side operation. This needs to have all the // correct remote endpoint info in order to instantiate the // correctly-sharded target cache. auto const storage_config = CreateStorageConfig(arguments.endpoint, arguments.protocol.hash_type, remote_exec_config->remote_address, remote_exec_config->platform_properties, remote_exec_config->dispatch); #else // For bootstrapping the TargetCache sharding is not needed, so we can // default all execution arguments. auto const storage_config = CreateStorageConfig( arguments.endpoint, arguments.protocol.hash_type); #endif // BOOTSTRAP_BUILD_TOOL if (not storage_config) { if (profile != nullptr) { profile->Write(kExitBuildEnvironment); } return kExitBuildEnvironment; } auto const storage = Storage::Create(&*storage_config); auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs : arguments.common.jobs; auto stage_args = arguments.cmd == SubCommand::kInstall or arguments.cmd == SubCommand::kInstallCas or arguments.cmd == SubCommand::kTraverse ? std::make_optional(std::move(arguments.stage)) : std::nullopt; auto rebuild_args = arguments.cmd == SubCommand::kRebuild ? std::make_optional(std::move(arguments.rebuild)) : std::nullopt; // statistics and progress instances; need to be kept alive // used also in bootstrapped just Statistics stats{}; Progress progress{}; #ifndef BOOTSTRAP_BUILD_TOOL // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &*local_exec_config, .storage_config = &*storage_config, .storage = &storage}; // pack the remote context instances to be passed as needed RemoteContext const remote_context{.auth = &*auth_config, .retry_config = &*retry_config, .exec_config = &*remote_exec_config}; auto const main_apis = ApiBundle::Create(&local_context, &remote_context, &repo_config); ExecutionContext const exec_context{ .repo_config = &repo_config, .apis = &main_apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = profile != nullptr ? std::make_optional(profile.get()) : std::nullopt}; const GraphTraverser::CommandLineArguments traverse_args{ jobs, std::move(arguments.build), std::move(stage_args), std::move(rebuild_args)}; GraphTraverser const traverser{ traverse_args, &exec_context, ProgressReporter::Reporter(&stats, &progress)}; if (arguments.cmd == SubCommand::kInstallCas) { if (not repo_config.SetGitCAS(storage_config->GitRoot(), &(*storage_config), LogLevel::Trace)) { Logger::Log(LogLevel::Debug, "Failed set Git CAS {}.", storage_config->GitRoot().string()); } return FetchAndInstallArtifacts( main_apis, arguments.fetch, remote_context) ? kExitSuccess : kExitBuildEnvironment; } if (arguments.cmd == SubCommand::kAddToCas) { return AddArtifactsToCas(arguments.to_add, storage, main_apis) ? kExitSuccess : kExitBuildEnvironment; } #endif // BOOTSTRAP_BUILD_TOOL auto [main_repo, main_ws_root] = DetermineRoots(&*storage_config, &repo_config, arguments.common, arguments.analysis); std::size_t eval_root_jobs = std::lround(std::ceil(std::sqrt(arguments.common.jobs))); #ifndef BOOTSTRAP_BUILD_TOOL std::optional serve = ServeApi::Create( *serve_config, &local_context, &remote_context, &main_apis); if (not EvaluatePrecomputedRoots(&repo_config, main_repo, serve ? &*serve : nullptr, *storage_config, traverse_args, &exec_context, eval_root_jobs)) { if (profile != nullptr) { profile->Write(kExitAnalysisFailure); } return kExitAnalysisFailure; } #else std::optional serve; #endif // BOOTSTRAP_BUILD_TOOL #ifndef BOOTSTRAP_BUILD_TOOL auto lock = GarbageCollector::SharedLock(*storage_config); if (not lock) { if (profile != nullptr) { profile->Write(kExitBuildEnvironment); } return kExitBuildEnvironment; } if (arguments.cmd == SubCommand::kTraverse) { if (arguments.graph.git_cas) { if (not ProtocolTraits::IsNative( arguments.protocol.hash_type)) { Logger::Log(LogLevel::Error, "Command line options {} and {} cannot be used " "together.", "--git-cas", "--compatible"); return kExitSyntaxError; } if (not repo_config.SetGitCAS(*arguments.graph.git_cas, &(*storage_config), LogLevel::Debug)) { Logger::Log(LogLevel::Warning, "Failed set Git CAS {}.", arguments.graph.git_cas->string()); } } if (traverser.BuildAndStage(arguments.graph.graph_file, arguments.graph.artifacts)) { return kExitSuccess; } return kExitFailure; } if (arguments.cmd == SubCommand::kDescribe) { if (auto id = DetermineNonExplicitTarget(main_repo, main_ws_root, &repo_config, arguments.analysis)) { auto result = arguments.describe.describe_rule ? DescribeUserDefinedRule(id->target, &repo_config, arguments.common.jobs, arguments.describe.print_json) : DescribeTarget(*id, &repo_config, serve, main_apis, arguments.common.jobs, arguments.describe.print_json); if (profile != nullptr) { profile->Write(result); } return result; } if (profile != nullptr) { profile->Write(kExitAnalysisFailure); } return kExitAnalysisFailure; } #endif // BOOTSTRAP_BUILD_TOOL auto id = ReadConfiguredTarget( main_repo, main_ws_root, &repo_config, arguments.analysis); if (profile != nullptr) { profile->SetTarget(id.target.ToJson()); profile->SetConfiguration(id.config.ToJson()); } auto serve_errors = nlohmann::json::array(); std::mutex serve_errors_access{}; BuildMaps::Target::ServeFailureLogReporter collect_serve_errors = [&serve_errors, &serve_errors_access]( BuildMaps::Target::ConfiguredTarget const& target, std::string const& blob) { std::unique_lock lock(serve_errors_access); auto target_desc = nlohmann::json::array(); target_desc.push_back(target.target.ToJson()); target_desc.push_back(target.config.ToJson()); auto entry = nlohmann::json::array(); entry.push_back(target_desc); entry.push_back(blob); serve_errors.push_back(entry); }; // create progress tracker for export targets Progress exports_progress{}; AnalyseContext analyse_ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto analyse_result = AnalyseTarget(&analyse_ctx, id, arguments.common.jobs, arguments.analysis.request_action_input, /*logger=*/nullptr, &collect_serve_errors, profile.get()); if (arguments.analysis.serve_errors_file) { Logger::Log(serve_errors.empty() ? LogLevel::Debug : LogLevel::Info, "Dumping serve-error information to {}", arguments.analysis.serve_errors_file->string()); std::ofstream os(*arguments.analysis.serve_errors_file); os << serve_errors.dump() << std::endl; } if (analyse_result) { Logger::Log(LogLevel::Info, "Analysed target {}", analyse_result->id.ToShortString( Evaluator::GetExpressionLogLimit())); { auto cached = stats.ExportsCachedCounter(); auto served = stats.ExportsServedCounter(); auto uncached = stats.ExportsUncachedCounter(); auto not_eligible = stats.ExportsNotEligibleCounter(); Logger::Log( served + cached + uncached + not_eligible > 0 ? LogLevel::Info : LogLevel::Debug, "Export targets found: {} cached, {}{} uncached, " "{} not eligible for caching", cached, served > 0 ? fmt::format("{} served, ", served) : "", uncached, not_eligible); } auto const artifacts_runfiles = ReadOutputArtifacts(analyse_result->target); auto const& [artifacts, runfiles] = artifacts_runfiles; auto dump_and_cleanup = [&]() { analyse_result->result_map.ToFile( arguments.analysis.graph_file, &stats, &progress); analyse_result->result_map.ToFile( arguments.analysis.graph_file_plain, &stats, &progress); for (auto const& to_build_file : arguments.analysis.artifacts_to_build_files) { DumpArtifactsToBuild(artifacts_runfiles.first, artifacts_runfiles.second, to_build_file); } // Clean up in parallel { TaskSystem ts{arguments.common.jobs}; analyse_result->result_map.Clear(&ts); } }; if (arguments.cmd == SubCommand::kAnalyse) { DiagnoseResults(*analyse_result, arguments.diagnose); dump_and_cleanup(); ReportTaintedness(*analyse_result); if (profile != nullptr) { profile->Write(kExitSuccess); } return kExitSuccess; } #ifndef BOOTSTRAP_BUILD_TOOL ReportTaintedness(*analyse_result); auto [actions, blobs, trees, tree_overlays] = analyse_result->result_map.ToResult(&stats, &progress); // collect cache targets and artifacts for target-level caching auto const cache_targets = analyse_result->result_map.CacheTargets(); auto cache_artifacts = CollectNonKnownArtifacts(cache_targets); std::thread dump_and_cleanup_thread(dump_and_cleanup); Logger::Log( LogLevel::Info, "{}ing{} {}.", arguments.cmd == SubCommand::kRebuild ? "Rebuild" : "Build", analyse_result->modified ? fmt::format(" input of action {} of", *(analyse_result->modified)) : "", analyse_result->id.ToShortString( Evaluator::GetExpressionLogLimit())); if (profile != nullptr) { profile->StartBuild(); } auto build_result = traverser.BuildAndStage(artifacts, runfiles, std::move(actions), std::move(blobs), std::move(trees), std::move(tree_overlays), std::move(cache_artifacts)); if (profile != nullptr) { profile->StopBuild(); } dump_and_cleanup_thread.join(); if (build_result) { WriteTargetCacheEntries( cache_targets, build_result->extra_infos, jobs, main_apis, arguments.tc.target_cache_write_strategy, storage.TargetCache(), nullptr, arguments.serve.remote_serve_address ? LogLevel::Performance : LogLevel::Warning); // Repeat taintedness message to make the user aware that // the artifacts are not for production use. ReportTaintedness(*analyse_result); if (build_result->failed_artifacts) { Logger::Log(LogLevel::Warning, "Build result contains failed artifacts."); } auto result = build_result->failed_artifacts ? kExitSuccessFailedArtifacts : kExitSuccess; if (profile != nullptr) { profile->Write(result); } return result; } if (profile != nullptr) { profile->Write(kExitFailure); } return kExitFailure; #endif // BOOTSTRAP_BUILD_TOOL } else { if (profile != nullptr) { profile->Write(kExitAnalysisFailure); } return kExitAnalysisFailure; } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Caught exception with message: {}", ex.what()); } if (profile != nullptr) { profile->Write(kExitBuildEnvironment); } return kExitBuildEnvironment; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/retry.cpp000066400000000000000000000024721516554100600252050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/retry.hpp" #include // std::move #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" [[nodiscard]] auto CreateRetryConfig(RetryArguments const& args) -> std::optional { RetryConfig::Builder builder; auto config = builder.SetInitialBackoffSeconds(args.initial_backoff_seconds) .SetMaxBackoffSeconds(args.max_backoff_seconds) .SetMaxAttempts(args.max_attempts) .Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/retry.hpp000066400000000000000000000017311516554100600252070ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_RETRY_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_RETRY_HPP #include #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/retry_cli.hpp" [[nodiscard]] auto CreateRetryConfig(RetryArguments const& args) -> std::optional; #endif // INCLUDED_SRC_BUILDOOL_MAIN_RETRY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/serve.cpp000066400000000000000000000567751516554100600252030ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/serve.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/location.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/buildtool/main/exit_codes.hpp" #include "src/utils/cpp/expected.hpp" namespace { /// \brief Overlay for ReadLocationObject accepting an ExpressionPtr and no /// workspace root and that can std::exit [[nodiscard]] auto ReadLocation(ExpressionPtr const& location) -> std::optional> { if (location.IsNotNull()) { auto res = ReadLocationObject(location->ToJson(), std::nullopt); if (not res) { Logger::Log(LogLevel::Error, res.error()); std::exit(kExitFailure); } return *res; } return std::nullopt; } } // namespace [[nodiscard]] auto ParseRetryCliOptions( Configuration const& config, gsl::not_null const& rargs) noexcept -> bool { auto max_attempts = config["max-attempts"]; if (max_attempts.IsNotNull()) { if (not max_attempts->IsNumber()) { Logger::Log( LogLevel::Error, "Invalid value for \"max-attempts\" {}. It must be a number.", max_attempts->ToString()); return false; } rargs->max_attempts = static_cast(max_attempts->Number()); } auto initial_backoff = config["initial-backoff-seconds"]; if (initial_backoff.IsNotNull()) { if (not initial_backoff->IsNumber()) { Logger::Log(LogLevel::Error, "Invalid value \"initial-backoff-seconds\" {}. It must " "be a number.", initial_backoff->ToString()); return false; } rargs->initial_backoff_seconds = static_cast(initial_backoff->Number()); } auto max_backoff = config["max-backoff-seconds"]; if (max_backoff.IsNotNull()) { if (not max_backoff->IsNumber()) { Logger::Log(LogLevel::Error, "Invalid value for \"max-backoff-seconds\" {}. It must " "be a number.", max_backoff->ToString()); return false; } rargs->max_backoff_seconds = static_cast(max_backoff->Number()); } return true; } void ReadJustServeConfig(gsl::not_null const& clargs) { Configuration serve_config{}; auto serve_path = clargs->serve.config; if (not FileSystemManager::ResolveSymlinks(&serve_path)) { return; } if (FileSystemManager::IsFile(serve_path)) { // json::parse may throw try { std::ifstream fs(clargs->serve.config); auto map = Expression::FromJson(nlohmann::json::parse(fs)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nExpected an " "object but found:\n{}", clargs->serve.config.string(), map->ToString()); std::exit(kExitFailure); } serve_config = Configuration{map}; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing serve service config file {} as JSON failed " "with error:\n{}", clargs->serve.config.string(), e.what()); std::exit(kExitFailure); } } else { Logger::Log(LogLevel::Error, "Cannot read serve service config file {}", clargs->serve.config.string()); std::exit(kExitFailure); } // read local build root auto local_root = ReadLocation(serve_config["local build root"]); if (local_root) { clargs->endpoint.local_root = local_root->first; } // read paths of additional lookup repositories auto repositories = serve_config["repositories"]; if (repositories.IsNotNull()) { if (not repositories->IsList()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"repositories\" has to be a list, but found {}", clargs->serve.config.string(), repositories->ToString()); std::exit(kExitFailure); } auto const& repos_list = repositories->List(); clargs->serve.repositories.reserve(clargs->serve.repositories.size() + repos_list.size()); for (auto const& repo : repos_list) { auto repo_root = ReadLocation(repo); if (repo_root) { if (not FileSystemManager::IsDirectory(repo_root->first)) { // warn the user that the path might not exist or is not // a directory Logger::Log(LogLevel::Warning, "In serve service config file {}:\nProvided " "known repository path {} does not exist or " "is not a directory!", clargs->serve.config.string(), repo_root->first.string()); } clargs->serve.repositories.emplace_back(repo_root->first); } } } // read logging arguments auto logging = serve_config["logging"]; if (logging.IsNotNull()) { if (not logging->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"logging\" has to be a map, but found {}", clargs->serve.config.string(), logging->ToString()); std::exit(kExitFailure); } // read in first the append flag auto append = logging->Get("append", Expression::none_t{}); if (append.IsNotNull()) { if (not append->IsBool()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for logging key " "\"append\" has to be a flag, but found {}", clargs->serve.config.string(), append->ToString()); std::exit(kExitFailure); } clargs->log.log_append = append->Bool(); } // read in the plain flag auto plain = logging->Get("plain", Expression::none_t{}); if (plain.IsNotNull()) { if (not plain->IsBool()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for logging key " "\"plain\" has to be a flag, but found {}", clargs->serve.config.string(), plain->ToString()); std::exit(kExitFailure); } clargs->log.plain_log = plain->Bool(); } // read in files field auto files = logging->Get("files", Expression::none_t{}); if (files.IsNotNull()) { if (not files->IsList()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for logging key " "\"files\" has to be a list, but found {}", clargs->serve.config.string(), files->ToString()); std::exit(kExitFailure); } auto const& files_list = files->List(); clargs->log.log_files.reserve(clargs->log.log_files.size() + files_list.size()); for (auto const& log_file : files_list) { auto path = ReadLocation(log_file); if (path) { clargs->log.log_files.emplace_back(path->first); } } } // read in limit field auto limit = logging->Get("limit", Expression::none_t{}); if (limit.IsNotNull()) { if (not limit->IsNumber()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for logging key " "\"limit\" has to be numeric, but found {}", clargs->serve.config.string(), limit->ToString()); std::exit(kExitFailure); } clargs->log.log_limit = ToLogLevel(limit->Number()); } // read stderr restriction auto stderr_limit = logging->Get("restrict stderr limit", Expression::none_t{}); if (stderr_limit.IsNotNull()) { if (not stderr_limit->IsNumber()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for logging key " "\"restrict stderr limit\" has to be numeric, but found {}", clargs->serve.config.string(), limit->ToString()); std::exit(kExitFailure); } clargs->log.restrict_stderr_log_limit = ToLogLevel(stderr_limit->Number()); } // read stderr restriction } // read client TLS authentication arguments auto auth_args = serve_config["authentication"]; if (auth_args.IsNotNull()) { if (not auth_args->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"authentication\" has to be a map, but found {}", clargs->serve.config.string(), auth_args->ToString()); std::exit(kExitFailure); } // read the TLS CA certificate auto cacert = ReadLocation(auth_args->Get("ca cert", Expression::none_t{})); if (cacert) { clargs->auth.tls_ca_cert = cacert->first; } // read the TLS client certificate auto client_cert = ReadLocation(auth_args->Get("client cert", Expression::none_t{})); if (client_cert) { clargs->cauth.tls_client_cert = client_cert->first; } // read the TLS client key auto client_key = ReadLocation(auth_args->Get("client key", Expression::none_t{})); if (client_key) { clargs->cauth.tls_client_key = client_key->first; } } // read remote service arguments auto remote_service = serve_config["remote service"]; if (remote_service.IsNotNull()) { if (not remote_service->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"remote service\" has to be a map, but found {}", clargs->serve.config.string(), remote_service->ToString()); std::exit(kExitFailure); } // read the interface auto interface = remote_service->Get("interface", Expression::none_t{}); if (interface.IsNotNull()) { if (not interface->IsString()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for " "remote service key \"interface\" has to be a " "string, but found {}", clargs->serve.config.string(), interface->ToString()); std::exit(kExitFailure); } clargs->service.interface = interface->String(); } // read the port auto port = remote_service->Get("port", Expression::none_t{}); if (port.IsNotNull()) { if (not port->IsNumber()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for remote " "service key \"port\" has to be numeric, but found {}", clargs->serve.config.string(), port->ToString()); std::exit(kExitFailure); } double val{}; if (std::modf(port->Number(), &val) != 0.0) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for remote " "service key \"port\" has to be an integer, but found {}", interface->ToString()); std::exit(kExitFailure); } // we are sure now that the port is an integer clargs->service.port = std::nearbyint(val); } // read the pid file auto pid_file = ReadLocation(remote_service->Get("pid file", Expression::none_t{})); if (pid_file) { clargs->service.pid_file = pid_file->first; } // read the info file auto info_file = ReadLocation( remote_service->Get("info file", Expression::none_t{})); if (info_file) { clargs->service.info_file = info_file->first; } // read the TLS server certificate auto server_cert = ReadLocation( remote_service->Get("server cert", Expression::none_t{})); if (server_cert) { clargs->sauth.tls_server_cert = server_cert->first; } // read the TLS server key auto server_key = ReadLocation( remote_service->Get("server key", Expression::none_t{})); if (server_key) { clargs->sauth.tls_server_key = server_key->first; } } // read execution endpoint arguments auto exec_endpoint = serve_config["execution endpoint"]; if (exec_endpoint.IsNotNull()) { if (not exec_endpoint->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nvalue for key " "\"execution endpoint\" has to be a map, but found {}", clargs->serve.config.string(), exec_endpoint->ToString()); std::exit(kExitFailure); } // read the compatible flag auto compatible = exec_endpoint->Get("compatible", Expression::none_t{}); if (compatible.IsNotNull()) { if (not compatible->IsBool()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for " "execution endpoint key \"compatible\" has to be a " "flag, but found {}", clargs->serve.config.string(), compatible->ToString()); std::exit(kExitFailure); } // compatibility is set immediately if flag is true if (compatible->Bool()) { clargs->protocol.hash_type = HashFunction::Type::PlainSHA256; } } // read the address auto address = exec_endpoint->Get("address", Expression::none_t{}); if (address.IsNotNull()) { if (not address->IsString()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for execution " "endpoint key \"address\" has to be a string, but found {}", clargs->serve.config.string(), address->ToString()); std::exit(kExitFailure); } clargs->endpoint.remote_execution_address = address->String(); } // read the instance name auto instance_name = exec_endpoint->Get("instance name", Expression::none_t{}); if (instance_name.IsNotNull()) { if (not instance_name->IsString()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\n" "Value for execution endpoint key \"instance " "name\" has to be a string, but found {}", clargs->serve.config.string(), instance_name->ToString()); std::exit(kExitFailure); } clargs->endpoint.remote_instance_name = instance_name->String(); } // read the address used by the client auto client_address = exec_endpoint->Get("client address", Expression::none_t{}); if (client_address.IsNotNull()) { if (not client_address->IsString()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for execution " "endpoint key \"client address\" has to be a string, but " "found {}", clargs->serve.config.string(), client_address->ToString()); std::exit(kExitFailure); } clargs->serve.client_remote_address = client_address->String(); } if (not ParseRetryCliOptions(serve_config, &clargs->retry)) { std::exit(kExitFailure); } } // read jobs value auto jobs = serve_config["jobs"]; if (jobs.IsNotNull()) { if (not jobs->IsNumber()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"jobs\" has to be a number, but found {}", clargs->serve.config.string(), jobs->ToString()); std::exit(kExitFailure); } clargs->common.jobs = static_cast(jobs->Number()); } // read build options auto build_args = serve_config["build"]; if (build_args.IsNotNull()) { if (not build_args->IsMap()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for key " "\"build\" has to be a map, but found {}", clargs->serve.config.string(), build_args->ToString()); std::exit(kExitFailure); } // read the build jobs auto build_jobs = build_args->Get("build jobs", Expression::none_t{}); if (build_jobs.IsNotNull()) { if (not build_jobs->IsNumber()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for build key " "\"build jobs\" has to be a number, but found {}", clargs->serve.config.string(), build_jobs->ToString()); std::exit(kExitFailure); } clargs->build.build_jobs = static_cast(build_jobs->Number()); } else { clargs->build.build_jobs = clargs->common.jobs; } // read action timeout auto timeout = build_args->Get("action timeout", Expression::none_t{}); if (timeout.IsNotNull()) { if (not timeout->IsNumber()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for build key " "\"action timeout\" has to be a number, but found {}", clargs->serve.config.string(), timeout->ToString()); std::exit(kExitFailure); } clargs->build.timeout = static_cast(timeout->Number()) * std::chrono::seconds{1}; } // read target-cache writing strategy auto strategy = build_args->Get("target-cache write strategy", Expression::none_t{}); if (strategy.IsNotNull()) { if (not strategy->IsString()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for build " "key \"target-cache write strategy\" has " "to be a string, but found {}", clargs->serve.config.string(), strategy->ToString()); std::exit(kExitFailure); } auto s_value = ToTargetCacheWriteStrategy(strategy->String()); if (not s_value) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nBuild key " "\"target-cache write strategy\" has unknown value {}", clargs->serve.config.string(), strategy->ToString()); std::exit(kExitFailure); } clargs->tc.target_cache_write_strategy = *s_value; } // read local launcher auto launcher = build_args->Get("local launcher", Expression::none_t{}); if (launcher.IsNotNull()) { if (not launcher->IsList()) { Logger::Log( LogLevel::Error, "In serve service config file {}:\nValue for build key " "\"local launcher\" has to be a list, but found {}", clargs->serve.config.string(), launcher->ToString()); std::exit(kExitFailure); } std::vector launcher_list{}; for (auto const& entry : launcher->List()) { if (not entry->IsString()) { Logger::Log(LogLevel::Error, "In serve service config file {}:\nValue for " "build key \"local launcher\" has to be a list " "of string, but found {} with entry {}", clargs->serve.config.string(), launcher->ToString(), entry->ToString()); std::exit(kExitFailure); } launcher_list.emplace_back(entry->String()); } clargs->build.local_launcher = launcher_list; } } } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/serve.hpp000066400000000000000000000022201516554100600251600ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP #define INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP #ifndef BOOTSTRAP_BUILD_TOOL #include "gsl/gsl" #include "src/buildtool/main/cli.hpp" /// \brief Parse the "just serve" config file. /// While having a separate config file, almost all fields are already used by /// "just" itself, so we can populate the respective known command-line fields. void ReadJustServeConfig(gsl::not_null const& clargs); #endif // BOOTSTRAP_BUILD_TOOL #endif // INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/version.cpp000066400000000000000000000026121516554100600255210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/version.hpp" #include #include #include #include "nlohmann/json.hpp" #include "src/utils/cpp/json.hpp" auto version() -> std::string { static const std::size_t kMajor = 1; static const std::size_t kMinor = 6; static const std::size_t kRevision = 5; std::string suffix = std::string{}; #ifdef VERSION_EXTRA_SUFFIX suffix += VERSION_EXTRA_SUFFIX; #endif nlohmann::json version_info = {{"version", {kMajor, kMinor, kRevision}}, {"suffix", suffix}}; #ifdef SOURCE_DATE_EPOCH version_info["SOURCE_DATE_EPOCH"] = (std::size_t)SOURCE_DATE_EPOCH; #else version_info["SOURCE_DATE_EPOCH"] = nullptr; #endif return IndentOnlyUntilDepth(version_info, 2, 1, {}); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/main/version.hpp000066400000000000000000000014611516554100600255270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDOOL_MAIN_VERSION_HPP #define INCLUDED_SRC_BUILDOOL_MAIN_VERSION_HPP #include auto version() -> std::string; #endif // INCLUDED_SRC_BUILDOOL_MAIN_VERSION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/000077500000000000000000000000001516554100600254235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/TARGETS000066400000000000000000000042021516554100600264550ustar00rootroot00000000000000{ "task": { "type": ["@", "rules", "CC", "library"] , "name": ["task"] , "hdrs": ["task.hpp"] , "stage": ["src", "buildtool", "multithreading"] , "private-ldflags": ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"] } , "notification_queue": { "type": ["@", "rules", "CC", "library"] , "name": ["notification_queue"] , "hdrs": ["notification_queue.hpp"] , "deps": ["task", ["@", "gsl", "", "gsl"], ["src/utils/cpp", "atomic"]] , "stage": ["src", "buildtool", "multithreading"] } , "task_system": { "type": ["@", "rules", "CC", "library"] , "name": ["task_system"] , "hdrs": ["task_system.hpp"] , "srcs": ["task_system.cpp"] , "deps": ["notification_queue", ["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "multithreading"] , "private-deps": ["task"] } , "async_map_node": { "type": ["@", "rules", "CC", "library"] , "name": ["async_map_node"] , "hdrs": ["async_map_node.hpp"] , "deps": ["task", "task_system", ["@", "gsl", "", "gsl"], ["src/utils/cpp", "gsl"]] , "stage": ["src", "buildtool", "multithreading"] } , "async_map": { "type": ["@", "rules", "CC", "library"] , "name": ["async_map"] , "hdrs": ["async_map.hpp"] , "deps": ["async_map_node", "task_system", ["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "multithreading"] } , "async_map_consumer": { "type": ["@", "rules", "CC", "library"] , "name": ["async_map_consumer"] , "hdrs": ["async_map_consumer.hpp"] , "deps": ["async_map", "task_system", ["@", "gsl", "", "gsl"]] , "stage": ["src", "buildtool", "multithreading"] } , "atomic_value": { "type": ["@", "rules", "CC", "library"] , "name": ["atomic_value"] , "hdrs": ["atomic_value.hpp"] , "deps": [["src/utils/cpp", "atomic"]] , "stage": ["src", "buildtool", "multithreading"] } , "async_map_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["async_map_utils"] , "hdrs": ["async_map_utils.hpp"] , "deps": [ "async_map_consumer" , ["@", "fmt", "", "fmt"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "multithreading"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/async_map.hpp000066400000000000000000000110431516554100600301050ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP #include #include #include #include #include // unique_lock #include #include #include #include #include // std::make_pair to use std::unordered_map's emplace() #include #include "gsl/gsl" #include "src/buildtool/multithreading/async_map_node.hpp" #include "src/buildtool/multithreading/task_system.hpp" // Wrapper around map data structure for KeyT->AsyncMapNode that only // exposes the possibility to retrieve the node for a certain key, adding it in // case of the key not yet being present. Thread-safe. Map look-ups happen under // a shared lock, and only in the case that key needs to be added to the // underlying map we uniquely lock. This is the default map class used inside // AsyncMapConsumer template class AsyncMap { public: using Node = AsyncMapNode; // Nodes will be passed onto tasks. Nodes are owned by this map. Nodes are // alive as long as this map lives. using NodePtr = Node*; explicit AsyncMap(std::size_t jobs) : width_{ComputeWidth(jobs)} {} AsyncMap() = default; /// \brief Retrieve node for certain key. Key and new node are emplaced in /// the map in case that the key does not exist already. /// \returns shared pointer to the Node associated to given key [[nodiscard]] auto GetOrCreateNode(KeyT const& key) -> NodePtr { auto* node_or_null = GetNodeOrNullFromSharedMap(key); return node_or_null != nullptr ? node_or_null : AddKey(key); } [[nodiscard]] auto GetPendingKeys() const -> std::vector { std::vector keys{}; std::size_t s = 0; for (auto& i : map_) { s += i.size(); } keys.reserve(s); for (auto& i : map_) { for (auto const& [key, node] : i) { if (not node->IsReady()) { keys.emplace_back(key); } } } return keys; } void Clear(gsl::not_null const& ts) { for (std::size_t i = 0; i < width_; ++i) { ts->QueueTask([i, this]() { map_[i].clear(); }); } } private: constexpr static std::size_t kScalingFactor = 2; std::size_t width_{ComputeWidth(0)}; std::vector m_{width_}; std::vector>> map_{width_}; constexpr static auto ComputeWidth(std::size_t jobs) -> std::size_t { if (jobs <= 0) { // Non-positive indicates to use the default value return ComputeWidth( std::max(1U, std::thread::hardware_concurrency())); } return jobs * kScalingFactor + 1; } [[nodiscard]] auto GetNodeOrNullFromSharedMap(KeyT const& key) -> NodePtr { auto part = std::hash{}(key) % width_; std::shared_lock sl{m_[part]}; auto it_to_key_pair = map_[part].find(key); if (it_to_key_pair != map_[part].end()) { // we know if the key is in the map then // the pair {key, node} is read only return it_to_key_pair->second.get(); } return nullptr; } [[nodiscard]] auto AddKey(KeyT const& key) -> NodePtr { auto part = std::hash{}(key) % width_; std::unique_lock ul{m_[part]}; auto it_to_key_pair = map_[part].find(key); if (it_to_key_pair != map_[part].end()) { return it_to_key_pair->second.get(); } auto new_node = std::make_unique(key); bool unused{}; std::tie(it_to_key_pair, unused) = map_[part].emplace(std::make_pair(key, std::move(new_node))); return it_to_key_pair->second.get(); } }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/async_map_consumer.hpp000066400000000000000000000342571516554100600320340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP #include #include #include #include #include #include #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/multithreading/async_map.hpp" #include "src/buildtool/multithreading/task_system.hpp" using AsyncMapConsumerLogger = std::function; using AsyncMapConsumerLoggerPtr = std::shared_ptr; // Thread safe class that enables us to add tasks to the queue system that // depend on values being ready. Value constructors are only queued once per key // and tasks that depend on such values are only queued once the values are // ready. As template parameters, it takes the type that keys will have, the // type that their corresponding values will have and the type of the underlying // thread-safe associative container. The default thread-safe associative // container is AsyncMap and any substitute must have the same // public interface to be used in AsyncMapConsumer. template > class AsyncMapConsumer { public: using Node = typename Map::Node; using NodePtr = typename Map::NodePtr; using Setter = std::function; using SetterPtr = std::shared_ptr; using Logger = AsyncMapConsumerLogger; using LoggerPtr = AsyncMapConsumerLoggerPtr; using FailureFunction = std::function; using FailureFunctionPtr = std::shared_ptr; using Consumer = std::function const&)>; using ConsumerPtr = std::shared_ptr; using SubCaller = std::function const&, Consumer, LoggerPtr)>; using SubCallerPtr = std::shared_ptr; using ValueCreator = std::function const&, SetterPtr, LoggerPtr, SubCallerPtr, Key const&)>; explicit AsyncMapConsumer(ValueCreator vc, std::size_t jobs = 0) : value_creator_{std::make_shared(std::move(vc))}, map_{jobs} {} /// \brief Makes sure that the consumer will be executed once the values for /// all the keys are available, and that the value creators for those keys /// are queued (if they weren't queued already). /// \param[in] ts task system /// \param[in] keys keys for the values that consumer requires /// \param[in] consumer function-like object that takes a vector of values /// and returns void that will be queued to be called with the values /// associated to keys once they are ready /// \param[in] logger function-like object that takes a string and a bool /// indicating that the event was fatal and returns /// void. This will be passed around and can be used to report errors /// (possibly with side effects outside AsyncMapConsumer) in the value /// creator /// \param[in] fail function to call instead of the consumer if the /// creation of this node failed void ConsumeAfterKeysReady(gsl::not_null const& ts, std::vector const& keys, Consumer&& consumer, Logger&& logger, FailureFunction&& fail) { ConsumeAfterKeysReady( ts, std::nullopt, keys, std::move(consumer), std::make_shared(std::move(logger)), std::make_shared(std::move(fail))); } // Similar to the previous method, but without failure function void ConsumeAfterKeysReady(gsl::not_null const& ts, std::vector const& keys, Consumer&& consumer, Logger&& logger) { ConsumeAfterKeysReady(ts, std::nullopt, keys, std::move(consumer), std::make_shared(std::move(logger)), nullptr); } [[nodiscard]] auto GetPendingKeys() const -> std::vector { return map_.GetPendingKeys(); } // Returns call order of the first cycle found in the requests map. [[nodiscard]] auto DetectCycle() const -> std::optional> { auto const& requests = GetPendingRequests(); std::vector calls{}; std::unordered_set known{}; calls.resize(requests.size() + 1, Key{}); known.reserve(requests.size()); for (auto const& [caller, _] : requests) { if (DetectCycleForCaller(&calls, &known, requests, caller)) { return calls; } } return std::nullopt; } void Clear(gsl::not_null const& ts) { map_.Clear(ts); } private: using NodeRequests = std::unordered_map>; std::shared_ptr value_creator_; Map map_; mutable std::shared_mutex requests_m_; std::unordered_map requests_by_thread_; // Similar to previous methods, but in this case the logger and failure // function are already std::shared_ptr type. void ConsumeAfterKeysReady(gsl::not_null const& ts, std::optional const& consumer_id, std::vector const& keys, Consumer&& consumer, LoggerPtr&& logger, FailureFunctionPtr&& fail) { auto consumerptr = std::make_shared(std::move(consumer)); if (keys.empty()) { ts->QueueTask([consumerptr = std::move(consumerptr)]() { (*consumerptr)({}); }); return; } auto nodes = EnsureValuesEventuallyPresent(ts, keys, std::move(logger)); auto first_node = nodes->at(0); if (fail) { first_node->QueueOnFailure(ts, [fail]() { (*fail)(); }); } auto const queued = first_node->AddOrQueueAwaitingTask( ts, [ts, consumerptr, nodes = std::move(nodes), fail, this, consumer_id]() { QueueTaskWhenAllReady( ts, consumer_id, consumerptr, fail, nodes, 1); }); if (consumer_id and not queued) { RecordNodeRequest(*consumer_id, first_node); } } [[nodiscard]] auto EnsureValuesEventuallyPresent( gsl::not_null const& ts, std::vector const& keys, LoggerPtr&& logger) -> std::shared_ptr> { std::vector nodes{}; nodes.reserve(keys.size()); std::transform(std::begin(keys), std::end(keys), std::back_inserter(nodes), [this, ts, logger](Key const& key) { return EnsureValuePresent(ts, key, logger); }); return std::make_shared>(std::move(nodes)); } // Retrieves node from map associated to given key and queues its processing // task (i.e. a task that executes the value creator) to the task system. // Note that the node will only queue a processing task once. [[nodiscard]] auto EnsureValuePresent(gsl::not_null const& ts, Key const& key, LoggerPtr const& logger) -> NodePtr { auto node = map_.GetOrCreateNode(key); auto setterptr = std::make_shared([ts, node](Value&& value) { node->SetAndQueueAwaitingTasks(ts, std::move(value)); }); auto failptr = std::make_shared([node, ts]() { node->Fail(ts); }); auto subcallerptr = std::make_shared( [ts, failptr = std::move(failptr), this, key]( std::vector const& keys, Consumer&& consumer, LoggerPtr&& logger) { ConsumeAfterKeysReady(ts, key, keys, std::move(consumer), std::move(logger), FailureFunctionPtr{failptr}); }); auto wrapped_logger = std::make_shared([logger, node, ts](auto msg, auto fatal) { if (fatal) { node->Fail(ts); } (*logger)(msg, fatal); }); node->QueueOnceProcessingTask( ts, [vc = value_creator_, ts, key, setterptr = std::move(setterptr), wrapped_logger = std::move(wrapped_logger), subcallerptr = std::move(subcallerptr)]() { (*vc)(ts, setterptr, wrapped_logger, subcallerptr, key); }); return node; } // Queues tasks for each node making sure that the task that calls the // consumer on the values is only queued once all the values are ready void QueueTaskWhenAllReady( gsl::not_null const& ts, std::optional const& consumer_id, ConsumerPtr const& consumer, // NOLINTNEXTLINE(performance-unnecessary-value-param) FailureFunctionPtr const& fail, std::shared_ptr> const& nodes, std::size_t pos) { if (pos == nodes->size()) { ts->QueueTask([nodes, consumer]() { std::vector values{}; values.reserve(nodes->size()); std::transform( nodes->begin(), nodes->end(), std::back_inserter(values), [](NodePtr const& node) { return &node->GetValue(); }); (*consumer)(values); }); } else { auto current = nodes->at(pos); if (fail) { current->QueueOnFailure(ts, [fail]() { (*fail)(); }); } auto const queued = current->AddOrQueueAwaitingTask( ts, [ts, consumer, fail, nodes, pos, this, consumer_id]() { QueueTaskWhenAllReady( ts, consumer_id, consumer, fail, nodes, pos + 1); }); if (consumer_id and not queued) { RecordNodeRequest(*consumer_id, current); } } } void RecordNodeRequest(Key const& consumer_id, gsl::not_null const& node) { auto tid = std::this_thread::get_id(); std::shared_lock shared(requests_m_); auto local_requests_it = requests_by_thread_.find(tid); if (local_requests_it == requests_by_thread_.end()) { shared.unlock(); std::unique_lock lock(requests_m_); // create new requests map for thread requests_by_thread_[tid] = NodeRequests{{consumer_id, {node}}}; return; } // every thread writes to separate local requests map local_requests_it->second[consumer_id].emplace(node); } [[nodiscard]] auto GetPendingRequests() const -> NodeRequests { NodeRequests requests{}; std::unique_lock lock(requests_m_); for (auto const& [_, local_requests] : requests_by_thread_) { requests.reserve(requests.size() + local_requests.size()); for (auto const& [consumer, deps] : local_requests) { auto& nodes = requests[consumer]; std::copy_if( // filter out nodes that are ready by now deps.begin(), deps.end(), std::inserter(nodes, nodes.end()), [](auto const& node) { return not node->IsReady(); }); } } return requests; } [[nodiscard]] static auto DetectCycleForCaller( gsl::not_null*> const& calls, gsl::not_null*> const& known, NodeRequests const& requests, Key const& caller, std::size_t pos = 0) -> bool { if (not known->contains(caller)) { auto it = requests.find(caller); if (it != requests.end()) { (*calls)[pos++] = caller; for (auto const& dep : it->second) { auto const& dep_key = dep->GetKey(); auto last = calls->begin() + static_cast(pos); if (std::find(calls->begin(), last, dep_key) != last) { (*calls)[pos++] = dep_key; calls->resize(pos); return true; } if (DetectCycleForCaller( calls, known, requests, dep_key, pos)) { return true; } } } known->emplace(caller); } return false; } }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/async_map_node.hpp000066400000000000000000000161031516554100600311140ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/multithreading/task.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/utils/cpp/gsl.hpp" // Wrapper around Value to enable async access to it in a continuation-style // programming way template class AsyncMapNode { public: explicit AsyncMapNode(Key key) : key_{std::move(key)} {} /// \brief Set value and queue awaiting tasks to the task system under a /// unique lock. Awaiting tasks are cleared to ensure node does not hold /// (shared) ownership of any data related to the task once they are given /// to the task system /// \param[in] ts task system to which tasks will be queued /// \param[in] value value to set void SetAndQueueAwaitingTasks(gsl::not_null const& ts, Value&& value) { std::unique_lock lock{m_}; if (failed_) { // The node is failed already; no value can be set. return; } value_ = std::move(value); for (auto& task : awaiting_tasks_) { ts->QueueTask(std::move(task)); } // After tasks are queued we need to release them and any other // information we are keeping about the tasks awaiting_tasks_.clear(); failure_tasks_.clear(); } /// \brief If node is not marked as queued to be processed, task is queued /// to the task system. A task to process the node (that is, set its value) /// can only be queued once. Lock free /// \param[in] ts task system /// \param[in] task processing task. Function type must have /// operator()() template void QueueOnceProcessingTask(gsl::not_null const& ts, Function&& task) { // if node was already queued to be processed, nothing to do if (GetAndMarkQueuedToBeProcessed()) { return; } ts->QueueTask(std::forward(task)); } /// \brief Ensure task will be queued to the task system once the value of /// the node is ready. This operation is lock free once the value is ready /// before that node is uniquely locked while task is being added to /// awaiting tasks /// \param[in] ts task system /// \param[in] task task awaiting for value. Function type must have /// operator()() /// \returns boolean indicating whether task was immediately queued. template [[nodiscard]] auto AddOrQueueAwaitingTask( gsl::not_null const& ts, Function&& task) -> bool { if (IsReady()) { ts->QueueTask(std::forward(task)); return true; } { std::unique_lock ul{m_}; if (failed_) { // If the node is failed (and hence will never get ready), do // not queue any more tasks. return false; } // Check again in case the node was made ready after the lock-free // check by another thread if (IsReady()) { ts->QueueTask(std::forward(task)); return true; } awaiting_tasks_.emplace_back(std::forward(task)); return false; } } /// \brief Ensure task will be queued to the task system once the value of /// the node is ready. This operation is lock free once the value is ready /// before that node is uniquely locked while task is being added to /// awaiting tasks /// \param[in] ts task system /// \param[in] task task awaiting for value. Function type must have /// operator()() template void QueueOnFailure(gsl::not_null const& ts, Function&& task) { if (IsReady()) { // The node is ready, so it won't fail any more. return; } { std::unique_lock ul{m_}; if (failed_) { ts->QueueTask(std::forward(task)); } else { failure_tasks_.emplace_back(std::forward(task)); } } } /// \brief Mark the node as failed and schedule the cleanup tasks. /// \param[in] ts task system void Fail(gsl::not_null const& ts) { std::unique_lock ul{m_}; if (IsReady()) { // The node has a value already, so it can't be marked as failed any // more return; } if (failed_) { // The was already marked as failed and the failure handled. // So there is nothing more to do. return; } failed_ = true; // As the node will never become ready, we have to clean up all tasks // and schedule the failure tasks. for (auto& task : failure_tasks_) { ts->QueueTask(std::move(task)); } awaiting_tasks_.clear(); failure_tasks_.clear(); } // Not thread safe, do not use unless the value has been already set [[nodiscard]] auto GetValue() const& noexcept -> Value const& { // Will only be checked in debug build ExpectsAudit(value_.has_value()); return *value_; } [[nodiscard]] auto GetValue() && noexcept = delete; [[nodiscard]] auto GetKey() const& noexcept -> Key const& { return key_; } [[nodiscard]] auto GetKey() && noexcept -> Key { return std::move(key_); } [[nodiscard]] auto IsReady() const noexcept -> bool { return value_.has_value(); } private: Key key_; std::optional value_; std::vector awaiting_tasks_; std::vector failure_tasks_; std::mutex m_; std::atomic is_queued_to_be_processed_{false}; bool failed_{false}; /// \brief Sets node as queued to be processed /// \returns True if it was already queued to be processed, false /// otherwise /// Note: this is an atomic, lock-free operation [[nodiscard]] auto GetAndMarkQueuedToBeProcessed() noexcept -> bool { return std::atomic_exchange(&is_queued_to_be_processed_, true); } }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/async_map_utils.hpp000066400000000000000000000066071516554100600313370ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_UTILS_HPP #include #include #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" /// \brief Utility to detect and report cycles for an AsyncMap instance. /// \param name Human-readable string identifier related to the map or its use. /// \param map The AsyncMap instance. /// \param key_printer Callable returning key-specific identifier in string /// format. /// \returns The resulting cycle message as a string, or nullopt if no cycle /// detected. template [[nodiscard]] auto DetectAndReportCycle( std::string const& name, AsyncMapConsumer const& map, std::function key_printer) -> std::optional { using namespace std::string_literals; auto cycle = map.DetectCycle(); if (cycle) { bool found{false}; std::ostringstream oss{}; oss << fmt::format("Cycle detected in {}:", name) << std::endl; for (auto const& k : *cycle) { auto match = (k == cycle->back()); std::string prefix; if (match) { prefix = found ? "`-- "s : ".-> "s; } else { prefix = found ? "| "s : " "s; } oss << prefix << key_printer(k) << std::endl; found = found or match; } return oss.str(); } return std::nullopt; } /// \brief Utility to detect and report pending tasks for an AsyncMap instance. /// \param name Human-readable string identifier related to the map or its use. /// \param map The AsyncMap instance. /// \param key_printer Callable returning key-specific identifier in string /// format. /// \param logger Named logger, or nullptr to use global logger. template void DetectAndReportPending(std::string const& name, AsyncMapConsumer const& map, std::function key_printer, Logger const* logger = nullptr) { using namespace std::string_literals; auto keys = map.GetPendingKeys(); if (not keys.empty()) { std::ostringstream oss{}; oss << fmt::format("Internal error, failed to evaluate pending {}:", name) << std::endl; for (auto const& k : keys) { oss << " " << key_printer(k) << std::endl; } Logger::Log(logger, LogLevel::Error, "{}", oss.str()); } } #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/atomic_value.hpp000066400000000000000000000044221516554100600306060ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ATOMIC_VALUE_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ATOMIC_VALUE_HPP #include #include #include #include "src/utils/cpp/atomic.hpp" // Value that can be set and get atomically. Reset is not thread-safe. template class AtomicValue { public: AtomicValue() noexcept = default; AtomicValue(AtomicValue const& other) noexcept = delete; AtomicValue(AtomicValue&& other) noexcept : data_{other.data_.load()} {} ~AtomicValue() noexcept = default; auto operator=(AtomicValue const& other) noexcept = delete; auto operator=(AtomicValue&& other) noexcept = delete; // Atomically set value once and return its reference. If this method is // called multiple times concurrently, the setter is called only once. In // any case, this method blocks until the value is ready. [[nodiscard]] auto SetOnceAndGet( std::function const& setter) const& noexcept -> T const& { if (data_.load() == nullptr) { if (not load_.exchange(true)) { data_.store(std::make_shared(setter())); data_.notify_all(); } else { data_.wait(nullptr); } } return *data_.load(); } [[nodiscard]] auto SetOnceAndGet(std::function const& setter) && = delete; // Reset, not thread-safe! void Reset() noexcept { load_ = false; data_ = nullptr; } private: mutable std::atomic load_{false}; mutable atomic_shared_ptr data_{nullptr}; }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ATOMIC_VALUE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/notification_queue.hpp000066400000000000000000000130361516554100600320310ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP #include #include #include #include #include #include #include #include #include // std::forward #include "gsl/gsl" #include "src/buildtool/multithreading/task.hpp" // TODO(modernize): Remove pragma once underlying issue is solved #include "src/utils/cpp/atomic.hpp" // IWYU pragma: keep // Counter that can block the caller until it reaches zero. class WaitableZeroCounter { public: explicit WaitableZeroCounter(std::size_t init = 0) : count_{init} {} void Decrement() { std::shared_lock lock{mutex_}; if (--count_ == 0) { cv_.notify_all(); } } void Increment() { ++count_; } void WaitForZero() { while (not IsZero()) { // loop to protect against spurious wakeups std::unique_lock lock{mutex_}; cv_.wait(lock, [this] { return IsZero(); }); } } void Abort() { std::shared_lock lock{mutex_}; done_ = true; cv_.notify_all(); } private: std::shared_mutex mutex_; std::condition_variable_any cv_; std::atomic count_; std::atomic done_ = false; [[nodiscard]] auto IsZero() noexcept -> bool { return count_ == 0 or done_; } }; class NotificationQueue { public: explicit NotificationQueue( gsl::not_null const& total_workload) : total_workload_{total_workload} {} NotificationQueue(NotificationQueue const& other) = delete; NotificationQueue(NotificationQueue&& other) noexcept : queue_{std::move(other.queue_)}, done_{other.done_}, total_workload_{other.total_workload_} {} ~NotificationQueue() = default; [[nodiscard]] auto operator=(NotificationQueue const& other) -> NotificationQueue& = delete; [[nodiscard]] auto operator=(NotificationQueue&& other) -> NotificationQueue& = delete; // Blocks the thread until it's possible to pop or we are done. // Note that the lock releases ownership of the mutex while waiting // for the queue to have some element or for the notification queue // state to be set to "done". // Returns task popped or nullopt if no task was popped [[nodiscard]] auto pop() -> std::optional { std::unique_lock lock{mutex_}; auto there_is_something_to_pop_or_we_are_done = [&]() { return not queue_.empty() or done_; }; if (not there_is_something_to_pop_or_we_are_done()) { total_workload_->Decrement(); ready_.wait(lock, there_is_something_to_pop_or_we_are_done); total_workload_->Increment(); } if (queue_.empty()) { return std::nullopt; } auto t = std::move(queue_.front()); queue_.pop_front(); total_workload_->Decrement(); return t; } // Returns nullopt if the mutex is already locked or the queue is empty, // otherwise pops the front element of the queue and returns it [[nodiscard]] auto try_pop() -> std::optional { std::unique_lock lock{mutex_, std::try_to_lock}; if (not lock or queue_.empty()) { return std::nullopt; } auto t = std::move(queue_.front()); queue_.pop_front(); total_workload_->Decrement(); return t; } // Push task once the mutex is available (locking it until addition is // finished) template void push(FunctionType&& f) { total_workload_->Increment(); { std::unique_lock lock{mutex_}; queue_.emplace_back(std::forward(f)); } ready_.notify_one(); } // Returns false if mutex is locked without pushing the task, pushes task // and returns true otherwise template [[nodiscard]] auto try_push(FunctionType&& f) -> bool { { std::unique_lock lock{mutex_, std::try_to_lock}; if (not lock) { return false; } total_workload_->Increment(); queue_.emplace_back(std::forward(f)); } ready_.notify_one(); return true; } // Method to communicate to the notification queue that there will not be // any more queries. Queries after calling this method are not guarantied to // work as expected void done() { { std::unique_lock lock{mutex_}; done_ = true; } ready_.notify_all(); } private: std::deque queue_; bool done_{false}; std::mutex mutex_; std::condition_variable ready_; gsl::not_null total_workload_; }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/task.hpp000066400000000000000000000034221516554100600270770ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP #include #include // std::move class Task { public: using TaskFunc = std::function; Task() noexcept = default; // NOLINTNEXTLINE(modernize-pass-by-value) explicit Task(TaskFunc const& function) noexcept : f_{function} {} explicit Task(TaskFunc&& function) noexcept : f_{std::move(function)} {} void operator()() { f_(); } // To be able to discern whether the internal f_ has been set or not, // allowing us to write code such as: /* Task t; while (!t) { t = TryGetTaskFromQueue(); // (*) } t(); // (**) */ // (*) does `return Task();` or `return {};` if queue is empty or locked) // (**) we can now surely execute the task (and be sure it won't throw any // exception) (for the sake of the example, imagine we are sure that the // queue wasn't empty, otherwise this would be an infinite loop) explicit operator bool() const noexcept { return f_.operator bool(); } private: TaskFunc f_; }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/task_system.cpp000066400000000000000000000052461516554100600305040ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/task_system.hpp" #include #include "gsl/gsl" #include "src/buildtool/multithreading/task.hpp" TaskSystem::TaskSystem() : TaskSystem(std::thread::hardware_concurrency()) {} TaskSystem::TaskSystem(std::size_t number_of_threads) : thread_count_{std::max(std::size_t{1}, number_of_threads)}, total_workload_{thread_count_} { for (std::size_t index = 0; index < thread_count_; ++index) { queues_.emplace_back(&total_workload_); } for (std::size_t index = 0; index < thread_count_; ++index) { threads_.emplace_back([&, index]() { Run(index); }); } } TaskSystem::~TaskSystem() { Finish(); // wait for tasks to finish Shutdown(); // stop the threads for (auto& t : threads_) { t.join(); } } void TaskSystem::Shutdown() noexcept { shutdown_ = true; // Abort the workload counter in case a system shut down was requested while // the task queues (workload) are not yet empty and someone is still waiting // on this counter to become zero. total_workload_.Abort(); for (auto& q : queues_) { q.done(); } } void TaskSystem::Finish() noexcept { // When starting a new task system all spawned threads will immediately go // to sleep and wait for tasks. Even after adding some tasks, it can take a // while until the first thread wakes up. Therefore, we need to wait for the // total workload (number of active threads _and_ total number of queued // tasks) to become zero. total_workload_.WaitForZero(); } void TaskSystem::Run(std::size_t idx) { Expects(thread_count_ > 0); while (not shutdown_) { std::optional t{}; for (std::size_t i = 0; i < thread_count_; ++i) { t = queues_[(idx + i) % thread_count_].try_pop(); if (t) { break; } } // NOLINTNEXTLINE(clang-analyzer-core.DivideZero) t = t ? t : queues_[idx % thread_count_].pop(); if (not t or shutdown_) { break; } (*t)(); } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/multithreading/task_system.hpp000066400000000000000000000061001516554100600304770ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP #define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP #include #include #include #include #include // std::forward #include #include "src/buildtool/multithreading/notification_queue.hpp" class TaskSystem { public: // Constructors create as many threads as specified (or // std::thread::hardware_concurrency() many if not specified) running // `TaskSystem::Run(index)` on them, where `index` is their position in // `threads_` TaskSystem(); explicit TaskSystem(std::size_t number_of_threads); TaskSystem(TaskSystem const&) = delete; TaskSystem(TaskSystem&&) = delete; auto operator=(TaskSystem const&) -> TaskSystem& = delete; auto operator=(TaskSystem&&) -> TaskSystem& = delete; // Destructor calls sets to "done" all notification queues and joins the // threads. Note that joining the threads will wait until the Run method // they are running is finished ~TaskSystem(); // Queue a task. Task will be added to the first notification queue that is // found to be unlocked or, if none is found (after kNumberOfAttemps // iterations), to the one in `index+1` position waiting until it's // unlocked. template void QueueTask(FunctionType&& f) noexcept { auto idx = index_++; for (std::size_t i = 0; i < thread_count_ * kNumberOfAttempts; ++i) { if (queues_[(idx + i) % thread_count_].try_push( std::forward(f))) { return; } } queues_[idx % thread_count_].push(std::forward(f)); } [[nodiscard]] auto NumberOfThreads() const noexcept -> std::size_t { return thread_count_; } // Initiate shutdown, skip execution of pending tasks void Shutdown() noexcept; // Wait for all queues to become empty _and_ all tasks to finish. void Finish() noexcept; private: std::size_t const thread_count_{ std::max(1U, std::thread::hardware_concurrency())}; std::vector threads_; std::vector queues_; std::atomic index_{0}; std::atomic shutdown_ = false; WaitableZeroCounter total_workload_; static constexpr std::size_t kNumberOfAttempts = 5; void Run(std::size_t idx); }; #endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/profile/000077500000000000000000000000001516554100600240435ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/profile/TARGETS000066400000000000000000000011421516554100600250750ustar00rootroot00000000000000{ "profile": { "type": ["@", "rules", "CC", "library"] , "name": ["profile"] , "hdrs": ["profile.hpp"] , "srcs": ["profile.cpp"] , "deps": [ ["@", "json", "", "json"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/main", "cli"] ] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "profile"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/profile/profile.cpp000066400000000000000000000140671516554100600262170ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/profile/profile.hpp" #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/utils/cpp/expected.hpp" void Profile::Write(int exit_code) { profile_["stop time"] = std::time(nullptr); if (not actions_.empty()) { auto actions = nlohmann::json::object(); for (auto const& [k, v] : actions_) { auto entry = nlohmann::json::object(); entry["cached"] = v.cached; if (not v.cached) { entry["duration"] = v.duration; } if (v.exit_code != 0) { entry["exit code"] = v.exit_code; } entry["artifacts"] = v.artifacts; if (v.out) { entry["stdout"] = *v.out; } if (v.err) { entry["stderr"] = *v.err; } actions[k] = entry; } profile_["actions"] = actions; } profile_["exit code"] = exit_code; if (not analysis_errors_.empty()) { profile_["analysis errors"] = analysis_errors_; } std::ofstream os(*output_file_); os << profile_.dump(2) << std::endl; } void Profile::SetTarget(nlohmann::json target) { profile_["target"] = std::move(target); } void Profile::SetConfiguration(nlohmann::json configuration) { profile_["configuration"] = std::move(configuration); } void Profile::SetCLI(CommandLineArguments const& cli) { switch (cli.cmd) { case SubCommand::kDescribe: profile_["subcommand"] = "describe"; break; case SubCommand::kAnalyse: profile_["subcommand"] = "analyse"; break; case SubCommand::kBuild: profile_["subcommand"] = "build"; break; case SubCommand::kInstall: profile_["subcommand"] = "install"; break; case SubCommand::kRebuild: profile_["subcommand"] = "rebuild"; break; default: // We only log information on the commands that support profiling. return; } if (cli.analysis.target) { if (cli.analysis.target->is_array()) { profile_["subcommand args"] = *cli.analysis.target; } else { auto args = nlohmann::json::array(); args.push_back(*cli.analysis.target); profile_["subcommand args"] = args; } } else { profile_["subcommand args"] = nlohmann::json::array(); } } void Profile::NoteActionCompleted(std::string const& id, IExecutionResponse::Ptr const& response, std::string const& cwd) { if (not response) { return; } std::unique_lock lock{mutex_}; auto artifacts = response->Artifacts(); std::optional out = std::nullopt; std::optional err = std::nullopt; if (response->HasStdOut()) { auto action_out = response->StdOutDigest(); if (action_out) { out = action_out->hash(); } } if (response->HasStdErr()) { auto action_err = response->StdErrDigest(); if (action_err) { err = action_err->hash(); } } if (not artifacts) { actions_[id] = ActionData{ .cached = response->IsCached(), .duration = response->ExecutionDuration(), .exit_code = response->ExitCode(), .out = out, .err = err, .artifacts = std::unordered_map()}; } else { actions_[id] = ActionData{ .cached = response->IsCached(), .duration = response->ExecutionDuration(), .exit_code = response->ExitCode(), .out = out, .err = err, .artifacts = std::unordered_map( (*artifacts)->size())}; if (cwd.empty()) { // the typical case of empty cwd, avoid unnecessary calls for (auto const& [k, v] : **artifacts) { actions_[id].artifacts.emplace(k, v.digest.hash()); } } else { std::filesystem::path base{cwd}; for (auto const& [k, v] : **artifacts) { actions_[id].artifacts.emplace( (base / k).lexically_normal().string(), v.digest.hash()); } } } } void Profile::NoteAnalysisError(std::string const& error_message) { std::unique_lock lock{mutex_}; analysis_errors_.emplace_back(error_message); } void Profile::SetRemoteExecutionConfig(RemoteExecutionConfig const& config) { auto remote = nlohmann::json::object(); if (config.remote_address) { remote["address"] = config.remote_address->ToJson(); } remote["properties"] = config.platform_properties; auto dispatch = nlohmann::json::array(); for (auto const& dispatch_entry : config.dispatch) { auto entry = nlohmann::json::array(); entry.emplace_back(dispatch_entry.first); entry.emplace_back(dispatch_entry.second.ToJson()); dispatch.emplace_back(entry); } remote["dispatch"] = dispatch; profile_["remote"] = remote; } void Profile::StartBuild() { profile_["build start time"] = std::time(nullptr); } void Profile::StopBuild() { profile_["build stop time"] = std::time(nullptr); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/profile/profile.hpp000066400000000000000000000044631516554100600262230ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROFILE_PROFILE_HPP #define INCLUDED_SRC_BUILDTOOL_PROFILE_PROFILE_HPP #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/main/cli.hpp" class Profile { public: explicit Profile(std::string output_file, CommandLineArguments const& cli) : output_file_{std::move(output_file)} { profile_ = nlohmann::json::object(); profile_["start time"] = std::time(nullptr); SetCLI(cli); } void Write(int exit_code); void SetRemoteExecutionConfig(RemoteExecutionConfig const&); void SetTarget(nlohmann::json target); void SetConfiguration(nlohmann::json configuration); void NoteActionCompleted(std::string const& id, IExecutionResponse::Ptr const& response, std::string const& cwd); void NoteAnalysisError(std::string const& error_message); void StartBuild(); void StopBuild(); private: struct ActionData { bool cached; double duration; int exit_code; std::optional out; std::optional err; std::unordered_map artifacts; }; std::optional output_file_; nlohmann::json profile_; std::unordered_map actions_; std::vector analysis_errors_; std::mutex mutex_; void SetCLI(CommandLineArguments const& cli); }; #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/000077500000000000000000000000001516554100600263405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/TARGETS000066400000000000000000000042651516554100600274030ustar00rootroot00000000000000{ "progress": { "type": ["@", "rules", "CC", "library"] , "name": ["progress"] , "hdrs": ["progress.hpp"] , "stage": ["src", "buildtool", "progress_reporting"] , "deps": [ "task_tracker" , ["src/buildtool/build_engine/target_map", "configured_target"] ] } , "task_tracker": { "type": ["@", "rules", "CC", "library"] , "name": ["task_tracker"] , "hdrs": ["task_tracker.hpp"] , "stage": ["src", "buildtool", "progress_reporting"] , "deps": [ ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "progress_reporter": { "type": ["@", "rules", "CC", "library"] , "name": ["progress_reporter"] , "hdrs": ["progress_reporter.hpp"] , "srcs": ["progress_reporter.cpp"] , "stage": ["src", "buildtool", "progress_reporting"] , "deps": [ "base_progress_reporter" , "progress" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/logging", "logging"] ] , "private-deps": [ "task_tracker" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/logging", "log_level"] ] } , "base_progress_reporter": { "type": ["@", "rules", "CC", "library"] , "name": ["base_progress_reporter"] , "hdrs": ["base_progress_reporter.hpp"] , "srcs": ["base_progress_reporter.cpp"] , "stage": ["src", "buildtool", "progress_reporting"] , "private-deps": [ ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "exports_progress_reporter": { "type": ["@", "rules", "CC", "library"] , "name": ["exports_progress_reporter"] , "hdrs": ["exports_progress_reporter.hpp"] , "srcs": ["exports_progress_reporter.cpp"] , "stage": ["src", "buildtool", "progress_reporting"] , "deps": [ "base_progress_reporter" , "progress" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/logging", "logging"] ] , "private-deps": [ "task_tracker" , ["@", "fmt", "", "fmt"] , ["src/buildtool/logging", "log_level"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/base_progress_reporter.cpp000066400000000000000000000035271516554100600336330ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include #include #include #include #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" auto BaseProgressReporter::Reporter(std::function report) noexcept -> progress_reporter_t { return [report = std::move(report)](std::atomic* done, std::condition_variable* cv) { std::mutex m; std::unique_lock lock(m); std::int64_t delay = kStartDelayMillis; while (not *done) { cv->wait_for(lock, std::chrono::milliseconds(delay)); if (not *done) { try { report(); } catch (std::exception const& ex) { Logger::Log( LogLevel::Warning, "calling progress report function failed with:\n{}", ex.what()); // continue with progress reporting } } delay = delay * kDelayScalingFactorNumerator / kDelayScalingFactorDenominator; } }; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/base_progress_reporter.hpp000066400000000000000000000031511516554100600336310ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_BASE_PROGRESS_REPORTER_HPP #define INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_BASE_PROGRESS_REPORTER_HPP #include #include #include #include // Type of a progress reporter. The reporter may only block in such a way that // it return on a notification of the condition variable; moreover, it has to // exit once the boolean is true. using progress_reporter_t = std::function*, std::condition_variable*)>; class BaseProgressReporter { public: [[nodiscard]] static auto Reporter( std::function report) noexcept -> progress_reporter_t; private: constexpr static std::int64_t kStartDelayMillis = 3000; // Scaling is roughly sqrt(2) constexpr static std::int64_t kDelayScalingFactorNumerator = 99; constexpr static std::int64_t kDelayScalingFactorDenominator = 70; }; #endif // INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_BASE_PROGRESS_REPORTER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/exports_progress_reporter.cpp000066400000000000000000000044461516554100600344260ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/progress_reporting/exports_progress_reporter.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" auto ExportsProgressReporter::Reporter(gsl::not_null const& stats, gsl::not_null const& progress, bool has_serve, Logger const* logger) noexcept -> progress_reporter_t { return BaseProgressReporter::Reporter( [stats, progress, has_serve, logger]() { // get 'found' counter last to ensure we never undercount the amount // of work not yet done auto cached = stats->ExportsCachedCounter(); auto served = stats->ExportsServedCounter(); auto uncached = stats->ExportsUncachedCounter(); auto not_eligible = stats->ExportsNotEligibleCounter(); auto found = stats->ExportsFoundCounter(); auto active = progress->TaskTracker().Active(); auto sample = progress->TaskTracker().Sample(); auto msg = fmt::format( "Export targets: {} found [{} cached{}, {} analysed locally]", found, cached, has_serve ? fmt::format(", {} served", served) : "", uncached + not_eligible); if ((active > 0) and not sample.empty()) { msg = fmt::format( "{} ({}{})", msg, sample, active > 1 ? ", ..." : ""); } Logger::Log(logger, LogLevel::Progress, "{}", msg); }); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/exports_progress_reporter.hpp000066400000000000000000000030031516554100600344170ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_EXPORTS_PROGRESS_REPORTER_HPP #define INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_EXPORTS_PROGRESS_REPORTER_HPP #include "gsl/gsl" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/progress_reporting/progress.hpp" /// \brief Reporter for progress in analysing export targets class ExportsProgressReporter { public: [[nodiscard]] static auto Reporter(gsl::not_null const& stats, gsl::not_null const& progress, bool has_serve, Logger const* logger = nullptr) noexcept -> progress_reporter_t; }; #endif // INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_EXPORTS_PROGRESS_REPORTER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/progress.hpp000066400000000000000000000041611516554100600307170ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_HPP #define INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_HPP #include #include #include #include #include #include #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" class Progress { public: [[nodiscard]] auto TaskTracker() noexcept -> TaskTracker& { return task_tracker_; } // Return a reference to the origin map. It is the responsibility of the // caller to ensure that access only happens in a single-threaded context. [[nodiscard]] auto OriginMap() noexcept -> std::unordered_map< std::string, std::vector< std::pair>>& { return origin_map_; } // Return a reference to the output map. It is the responsibility of the // caller to ensure that access only happens in a single-threaded context. [[nodiscard]] auto OutputMap() noexcept -> std::unordered_map& { return output_map_; } private: ::TaskTracker task_tracker_{}; std::unordered_map< std::string, std::vector< std::pair>> origin_map_; std::unordered_map output_map_; }; #endif // INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/progress_reporter.cpp000066400000000000000000000072531516554100600326410ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/progress_reporting/progress_reporter.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" auto ProgressReporter::Reporter(gsl::not_null const& stats, gsl::not_null const& progress, Logger const* logger) noexcept -> progress_reporter_t { return BaseProgressReporter::Reporter([stats, progress, logger]() { int total = gsl::narrow(progress->OriginMap().size()); // Note: order matters; queued has to be queried last auto const& sample = progress->TaskTracker().Sample(); int cached = stats->ActionsCachedCounter(); int run = stats->ActionsExecutedCounter(); int queued = stats->ActionsQueuedCounter(); int active = queued - run - cached; std::string now_msg; if (active > 0 and not sample.empty()) { std::string output_info{}; auto const& output_map = progress->OutputMap(); auto output = output_map.find(sample); if (output != output_map.end()) { output_info = fmt::format( " {}", nlohmann::json(std::filesystem::path(output->second) .filename() .string()) .dump()); } auto const& origin_map = progress->OriginMap(); auto origins = origin_map.find(sample); if (origins != origin_map.end() and not origins->second.empty()) { auto const& origin = origins->second[0]; now_msg = fmt::format(" ({}#{}{}{})", origin.first.target.ToString(), origin.second, output_info, active > 1 ? ", ..." : ""); } else { now_msg = fmt::format(" ({}{}{})", nlohmann::json(sample).dump(), output_info, active > 1 ? ", ..." : ""); } } constexpr int kOneHundred{100}; int total_work = total - cached; int progress = kOneHundred; // default if no work has to be done if (total_work > 0) { progress = run * kOneHundred / total_work; } Logger::Log(logger, LogLevel::Progress, "[{:3}%] {} cached, {} run, {} processing{}.", progress, cached, run, active, now_msg); }); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/progress_reporter.hpp000066400000000000000000000025601516554100600326420ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP #define INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP #include "gsl/gsl" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/progress_reporting/progress.hpp" class ProgressReporter { public: [[nodiscard]] static auto Reporter(gsl::not_null const& stats, gsl::not_null const& progress, Logger const* logger = nullptr) noexcept -> progress_reporter_t; }; #endif // INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/progress_reporting/task_tracker.hpp000066400000000000000000000042521516554100600315310ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_TASK_TRACKER_HPP #define INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_TASK_TRACKER_HPP #include #include #include #include #include #include #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" class TaskTracker { public: auto Start(const std::string& id) noexcept -> void { std::unique_lock lock(m_); ++prio_; try { running_.emplace(id, prio_); } catch (...) { Logger::Log(LogLevel::Warning, "Internal error in progress tracking; progress reports " "might be incorrect."); } } auto Stop(const std::string& id) noexcept -> void { std::unique_lock lock(m_); running_.erase(id); } [[nodiscard]] auto Sample() noexcept -> std::string { std::unique_lock lock(m_); std::string result{}; std::uint64_t started = prio_ + 1; for (auto const& it : running_) { if (it.second < started) { result = it.first; started = it.second; } } return result; } [[nodiscard]] auto Active() noexcept -> std::size_t { std::unique_lock lock(m_); return running_.size(); } private: std::uint64_t prio_{}; std::mutex m_; std::unordered_map running_; }; #endif // INCLUDED_SRC_BUILDTOOL_PROGRESS_REPORTING_TASK_TRACKER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/000077500000000000000000000000001516554100600243605ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/000077500000000000000000000000001516554100600256535ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/TARGETS000066400000000000000000000107131516554100600267110ustar00rootroot00000000000000{ "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "hdrs": ["config.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/main", "build_utils"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "serve_api", "remote"] } , "source_tree_client": { "type": ["@", "rules", "CC", "library"] , "name": ["source_tree_client"] , "hdrs": ["source_tree_client.hpp"] , "srcs": ["source_tree_client.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] , "stage": ["src", "buildtool", "serve_api", "remote"] , "private-deps": [ ["@", "grpc", "", "grpc++"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/logging", "log_level"] ] } , "serve_api": { "type": ["@", "rules", "CC", "library"] , "name": ["serve_api"] , "hdrs": ["serve_api.hpp"] , "srcs": ["serve_api.cpp"] , "deps": [ "config" , "configuration_client" , "source_tree_client" , "target_client" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "buildtool", "serve_api", "remote"] } , "target_client": { "type": ["@", "rules", "CC", "library"] , "name": ["target_client"] , "hdrs": ["target_client.hpp"] , "srcs": ["target_client.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "storage"] ] , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] , "stage": ["src", "buildtool", "serve_api", "remote"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "expected"] ] } , "configuration_client": { "type": ["@", "rules", "CC", "library"] , "name": ["configuration_client"] , "hdrs": ["configuration_client.hpp"] , "srcs": ["configuration_client.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/logging", "logging"] ] , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] , "stage": ["src", "buildtool", "serve_api", "remote"] , "private-deps": [ ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/logging", "log_level"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/config.hpp000066400000000000000000000156131516554100600276370ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_CONFIG_HPP #include #include #include #include #include #include #include // needed by durations #include #include #include #include "fmt/core.h" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/utils/cpp/expected.hpp" struct RemoteServeConfig final { class Builder; // Server address of the serve endpoint. std::optional const remote_address; // Execution endpoint used by the client. std::optional const client_execution_address; // Known Git repositories to serve server. std::vector const known_repositories; // Number of jobs std::size_t const jobs = 0; // Number of build jobs std::size_t const build_jobs = 0; // Action timeout std::chrono::milliseconds const action_timeout{}; // Strategy for synchronizing target-level cache TargetCacheWriteStrategy const tc_strategy{TargetCacheWriteStrategy::Sync}; }; class RemoteServeConfig::Builder final { public: // Set remote execution and cache address, unsets if parsing `address` fails auto SetClientExecutionAddress(std::optional value) noexcept -> Builder& { client_execution_address_ = std::move(value); return *this; } // Set remote execution and cache address, unsets if parsing `address` fails auto SetRemoteAddress(std::optional value) noexcept -> Builder& { remote_address_ = std::move(value); return *this; } // Set the list of known repositories auto SetKnownRepositories(std::vector value) noexcept -> Builder& { known_repositories_ = std::move(value); return *this; } // Set the number of jobs auto SetJobs(std::size_t value) noexcept -> Builder& { jobs_ = value; return *this; } // Set the number of build jobs auto SetBuildJobs(std::size_t value) noexcept -> Builder& { build_jobs_ = value; return *this; } // Set the action timeout auto SetActionTimeout(std::chrono::milliseconds const& value) noexcept -> Builder& { action_timeout_ = value; return *this; } auto SetTCStrategy(TargetCacheWriteStrategy value) noexcept -> Builder& { tc_strategy_ = value; return *this; } /// \brief Finalize building and create RemoteServeConfig. /// \return RemoteServeConfig on success or an error on failure. [[nodiscard]] auto Build() const noexcept -> expected { // To not duplicate default arguments of RemoteServeConfig in builder, // create a default config and copy default arguments from there. RemoteServeConfig const default_config{}; auto remote_address = default_config.remote_address; if (remote_address_.has_value()) { remote_address = ParseAddress(*remote_address_); if (not remote_address) { return unexpected{ fmt::format("Setting serve service address '{}' failed.", *remote_address_)}; } } auto client_execution_address = default_config.client_execution_address; if (client_execution_address_.has_value()) { client_execution_address = ParseAddress(*client_execution_address_); if (not client_execution_address) { return unexpected{ fmt::format("Setting client execution address '{}' failed.", *client_execution_address_)}; } } auto known_repositories = default_config.known_repositories; if (known_repositories_.has_value()) { try { known_repositories = *known_repositories_; } catch (std::exception const& ex) { return unexpected{ std::string("Setting known repositories failed.")}; } } auto jobs = default_config.jobs; if (jobs_.has_value()) { jobs = *jobs_; if (jobs == 0) { return unexpected{std::string{"Setting jobs failed."}}; } } auto build_jobs = default_config.jobs; if (build_jobs_.has_value()) { build_jobs = *build_jobs_; if (build_jobs == 0) { return unexpected{std::string{"Setting build jobs failed."}}; } } auto action_timeout = default_config.action_timeout; if (action_timeout_.has_value()) { action_timeout = *action_timeout_; if (bool const valid = action_timeout > std::chrono::seconds{0}; not valid) { return unexpected{ std::string{"Setting action timeout failed."}}; } } auto tc_strategy = default_config.tc_strategy; if (tc_strategy_.has_value()) { tc_strategy = *tc_strategy_; } return RemoteServeConfig{ .remote_address = std::move(remote_address), .client_execution_address = std::move(client_execution_address), .known_repositories = std::move(known_repositories), .jobs = jobs, .build_jobs = build_jobs, .action_timeout = action_timeout, .tc_strategy = tc_strategy}; } private: // Server address of the serve endpoint. std::optional remote_address_; // Execution endpoint used by the client. std::optional client_execution_address_; // Known Git repositories to serve server. std::optional> known_repositories_; // Number of jobs std::optional jobs_; // Number of build jobs std::optional build_jobs_; // Action timeout std::optional action_timeout_; // Strategy for synchronizing target-level cache std::optional tc_strategy_; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/configuration_client.cpp000066400000000000000000000125551516554100600325740ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/configuration_client.hpp" #include #include #include #include #include #include "justbuild/just_serve/just_serve.pb.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/logging/log_level.hpp" ConfigurationClient::ConfigurationClient( ServerAddress address, gsl::not_null const& remote_context) noexcept : client_serve_address_{std::move(address)}, remote_config_{*remote_context->exec_config} { stub_ = justbuild::just_serve::Configuration::NewStub( CreateChannelWithCredentials(client_serve_address_.host, client_serve_address_.port, remote_context->auth)); } auto ConfigurationClient::CheckServeRemoteExecution() const noexcept -> bool { auto const client_remote_address = remote_config_.remote_address; if (not client_remote_address) { logger_.Emit(LogLevel::Error, "Internal error: the remote execution endpoint should " "have been set."); return false; } grpc::ClientContext context; justbuild::just_serve::RemoteExecutionEndpointRequest request{}; justbuild::just_serve::RemoteExecutionEndpointResponse response{}; grpc::Status status = stub_->RemoteExecutionEndpoint(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Error, status); return false; } try { auto client_msg = client_remote_address->ToJson().dump(); std::string serve_msg{}; if (response.address().empty()) { // just serve acts as just execute, so from the server's perspective // there is nothing to be checked and it's the client's job to // ensure that its remote execution and serve endpoints match // // NOTE: This check might make sense to be removed altogether in the // future, or updated to (at most) a warning. if (client_remote_address->ToJson() == client_serve_address_.ToJson()) { return true; } serve_msg = client_serve_address_.ToJson().dump(); } else { nlohmann::json serve_remote_endpoint{}; try { serve_remote_endpoint = nlohmann::json::parse(response.address()); } catch (std::exception const& ex) { logger_.Emit( LogLevel::Error, "Parsing configured address from response failed with:\n{}", ex.what()); } if (serve_remote_endpoint == client_remote_address->ToJson()) { if (remote_config_.remote_instance_name != response.remote_instance_name()) { // Warn if different values for remote_instance_name are // used, but don't error out. Most RBE services ignore the // instance name anyway, or have only one valid value. logger_.Emit( LogLevel::Warning, "Serve uses remote_instance_name {} while client uses " "{}", nlohmann::json(remote_config_.remote_instance_name) .dump(), nlohmann::json(response.remote_instance_name()).dump()); } return true; } serve_msg = serve_remote_endpoint.dump(); } // log any mismatch found logger_.Emit( LogLevel::Error, "Different execution endpoint detected!\nIn order to correctly use " "the serve service, its remote execution endpoint must be the same " "used by the client.\nserve remote endpoint: {}\nclient remote " "endpoint: {}", serve_msg, client_msg); } catch (std::exception const& ex) { logger_.Emit( LogLevel::Error, "parsing response for remote endpoint request failed with:\n{}", ex.what()); } return false; } auto ConfigurationClient::IsCompatible() const noexcept -> std::optional { grpc::ClientContext context; justbuild::just_serve::CompatibilityRequest request{}; justbuild::just_serve::CompatibilityResponse response{}; grpc::Status status = stub_->Compatibility(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Error, status); return std::nullopt; } return response.compatible(); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/configuration_client.hpp000066400000000000000000000034611516554100600325750ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/logging/logger.hpp" /// Implements client side for Configuration service defined in: /// src/buildtool/serve_api/serve_service/just_serve.proto class ConfigurationClient { public: explicit ConfigurationClient( ServerAddress address, gsl::not_null const& remote_context) noexcept; [[nodiscard]] auto CheckServeRemoteExecution() const noexcept -> bool; [[nodiscard]] auto IsCompatible() const noexcept -> std::optional; private: ServerAddress const client_serve_address_; RemoteExecutionConfig const& remote_config_; std::unique_ptr stub_; Logger logger_{"RemoteConfigurationClient"}; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/serve_api.cpp000066400000000000000000000147061516554100600303440ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/storage.hpp" auto ServeApi::UploadTree(ArtifactDigest const& tree, std::filesystem::path const& git_repo) const noexcept -> expected { static constexpr bool kIsSyncError = true; if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) { return unexpected{UploadError{ fmt::format("Not a git tree: {}", tree.hash()), not kIsSyncError}}; } // Set up the repository config; compatibility of used storage instance is // irrelevant here, as only the build root path info is needed. RepositoryConfig repo; if (not repo.SetGitCAS(git_repo, &storage_config_)) { return unexpected{UploadError{ fmt::format("Failed to SetGitCAS at {}", git_repo.string()), not kIsSyncError}}; } bool const with_rehashing = not ProtocolTraits::IsNative(storage_config_.hash_function.GetType()); // Create a native storage config if rehashing is needed: std::optional native_storage_config; if (with_rehashing) { auto config = StorageConfig::Builder::Rebuild(storage_config_) .SetHashType(HashFunction::Type::GitSHA1) .Build(); if (not config.has_value()) { return unexpected{ UploadError{fmt::format("Failed to create native storage: {}", std::move(config).error()), not kIsSyncError}}; } native_storage_config.emplace(*config); } std::shared_ptr git_api; if (with_rehashing) { git_api = std::make_shared( &repo, &*native_storage_config, &storage_config_, &*apis_.local); } else { git_api = std::make_shared(&repo, &storage_config_); } // Upload tree to remote CAS: if (not git_api->RetrieveToCas( {Artifact::ObjectInfo{tree, ObjectType::Tree}}, *apis_.remote)) { return unexpected{ UploadError{fmt::format("Failed to sync tree {} from repository {}", tree.hash(), git_repo.string()), not kIsSyncError}}; } ArtifactDigest on_remote = tree; // Read rehashed digest if needed: if (with_rehashing) { auto rehashed = RehashUtils::ReadRehashedDigest( tree, *native_storage_config, storage_config_, /*from_git=*/true); if (not rehashed.has_value()) { return unexpected{ UploadError{std::move(rehashed).error(), not kIsSyncError}}; } if (not rehashed.value().has_value()) { return unexpected{UploadError{ fmt::format("No digest provided to sync root tree {}", tree.hash()), kIsSyncError}}; } on_remote = rehashed.value()->digest; } // Ask serve to get tree from remote: if (not this->GetTreeFromRemote(on_remote)) { return unexpected{UploadError{ fmt::format("Serve endpoint failed to sync root tree {}.", tree.hash()), kIsSyncError}}; } return std::monostate{}; } auto ServeApi::DownloadTree(ArtifactDigest const& tree) const noexcept -> expected { if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) { return unexpected{fmt::format("Not a git tree: {}", tree.hash())}; } // Check the tree is already in native CAS: auto native_config = StorageConfig::Builder::Rebuild(storage_config_) .SetHashType(HashFunction::Type::GitSHA1) .Build(); if (not native_config.has_value()) { return unexpected{fmt::format("Failed to create native storage: {}", std::move(native_config).error())}; } if (Storage::Create(&*native_config).CAS().TreePath(tree).has_value()) { return std::monostate{}; } // Make tree available on the remote end point: auto const on_remote = TreeInRemoteCAS(tree.hash()); if (not on_remote.has_value()) { return unexpected{ fmt::format("Failed to upload {} from serve to the remote end " "point.", tree.hash())}; } // Download tree from the remote end point: Artifact::ObjectInfo const info{*on_remote, ObjectType::Tree}; if (not apis_.remote->RetrieveToCas({info}, *apis_.local)) { return unexpected{fmt::format( "Failed to download {} from the remote end point.", tree.hash())}; } // The remote end point may operate in the compatible mode. In such a case, // an extra rehashing is needed: if (not ProtocolTraits::IsNative(storage_config_.hash_function.GetType())) { auto rehashed = RehashUtils::RehashDigest( {info}, storage_config_, *native_config, &apis_); if (not rehashed.has_value() or rehashed->size() != 1 or rehashed->front().digest != tree) { return unexpected{fmt::format("Failed to rehash downloaded {}:\n{}", on_remote->hash(), std::move(rehashed).error())}; } } return std::monostate{}; } #endif just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/serve_api.hpp000066400000000000000000000211401516554100600303370ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_SERVE_API_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_SERVE_API_HPP #ifdef BOOTSTRAP_BUILD_TOOL class ServeApi final {}; #else #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/configuration_client.hpp" #include "src/buildtool/serve_api/remote/source_tree_client.hpp" #include "src/buildtool/serve_api/remote/target_client.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/expected.hpp" class ServeApi final { public: explicit ServeApi(ServerAddress const& address, gsl::not_null const& local_context, gsl::not_null const& remote_context, gsl::not_null const& apis) noexcept : stc_{address, local_context->storage_config->hash_function, remote_context}, tc_{address, local_context->storage, remote_context, apis}, cc_{address, remote_context}, storage_config_{*local_context->storage_config}, apis_{*apis} {} ~ServeApi() noexcept = default; ServeApi(ServeApi const&) = delete; ServeApi(ServeApi&&) = delete; auto operator=(ServeApi const&) -> ServeApi& = delete; auto operator=(ServeApi&&) -> ServeApi& = delete; [[nodiscard]] static auto Create( RemoteServeConfig const& serve_config, gsl::not_null const& local_context, gsl::not_null const& remote_context, gsl::not_null const& apis) noexcept -> std::optional { if (serve_config.remote_address) { return std::make_optional(*serve_config.remote_address, local_context, remote_context, apis); } return std::nullopt; } [[nodiscard]] auto RetrieveTreeFromCommit( std::string const& commit, std::string const& subdir = ".", bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeCommitTree(commit, subdir, sync_tree); } [[nodiscard]] auto RetrieveTreeFromArchive( std::string const& content, std::string const& archive_type = "archive", std::string const& subdir = ".", std::optional const& resolve_symlinks = std::nullopt, bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeArchiveTree( content, archive_type, subdir, resolve_symlinks, sync_tree); } [[nodiscard]] auto RetrieveTreeFromDistdir( std::shared_ptr> const& distfiles, bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeDistdirTree(distfiles, sync_tree); } [[nodiscard]] auto RetrieveTreeFromForeignFile( const std::string& content, const std::string& name, bool executable) const noexcept -> SourceTreeClient::result_t { return stc_.ServeForeignFileTree(content, name, executable); } [[nodiscard]] auto ContentInRemoteCAS(std::string const& content) const noexcept -> expected { return stc_.ServeContent(content); } [[nodiscard]] auto TreeInRemoteCAS(std::string const& tree_id) const noexcept -> expected { return stc_.ServeTree(tree_id); } [[nodiscard]] auto CheckRootTree(std::string const& tree_id) const noexcept -> std::optional { return stc_.CheckRootTree(tree_id); } [[nodiscard]] auto GetTreeFromRemote( ArtifactDigest const& digest) const noexcept -> bool { return stc_.GetRemoteTree(digest); } [[nodiscard]] auto ComputeTreeStructure(ArtifactDigest const& digest) const noexcept -> expected { return stc_.ComputeTreeStructure(digest); } [[nodiscard]] auto ServeTargetVariables(std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional> { return tc_.ServeTargetVariables(target_root_id, target_file, target); } [[nodiscard]] auto ServeTargetDescription(std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional { return tc_.ServeTargetDescription(target_root_id, target_file, target); } [[nodiscard]] auto ServeTarget(const TargetCacheKey& key, const ArtifactDigest& repo_key, bool keep_artifact_root = false) const noexcept -> std::optional { return tc_.ServeTarget(key, repo_key, keep_artifact_root); } [[nodiscard]] auto CheckServeRemoteExecution() const noexcept -> bool { return cc_.CheckServeRemoteExecution(); } [[nodiscard]] auto IsCompatible() const noexcept -> std::optional { return cc_.IsCompatible(); } class UploadError; /// \brief Upload a git tree from git repo to serve. /// \param tree Tree to upload. /// \param git_repo Git repository where the tree can be found. /// \return std::monostate if the tree is available for this serve /// instance after the call, or an unexpected UploadError on failure. [[nodiscard]] auto UploadTree(ArtifactDigest const& tree, std::filesystem::path const& git_repo) const noexcept -> expected; /// \brief Download a git tree from serve. /// \param tree Tree to download. /// \return std::monostate if after the call the requested tree can be found /// in the native CAS to which this serve instance is bound to, or an /// unexpected error message on failure. [[nodiscard]] auto DownloadTree(ArtifactDigest const& tree) const noexcept -> expected; private: // source tree service client SourceTreeClient const stc_; // target service client TargetClient const tc_; // configuration service client ConfigurationClient const cc_; StorageConfig const& storage_config_; ApiBundle const& apis_; }; class ServeApi::UploadError final { public: [[nodiscard]] auto Message() const& noexcept -> std::string const& { return message_; } [[nodiscard]] auto Message() && noexcept -> std::string { return std::move(message_); } [[nodiscard]] auto IsSyncError() const noexcept -> bool { return is_sync_error_; } private: friend ServeApi; explicit UploadError(std::string message, bool is_sync_error) noexcept : message_{std::move(message)}, is_sync_error_{is_sync_error} {} std::string message_; bool is_sync_error_; }; #endif // BOOTSTRAP_BUILD_TOOL #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_SERVE_API_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/source_tree_client.cpp000066400000000000000000000371211516554100600322400ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/source_tree_client.hpp" #include #include #include "justbuild/just_serve/just_serve.pb.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/logging/log_level.hpp" namespace { auto StringToArchiveType(std::string const& type) noexcept -> justbuild::just_serve::ServeArchiveTreeRequest_ArchiveType { using ServeArchiveType = justbuild::just_serve::ServeArchiveTreeRequest_ArchiveType; return type == "zip" ? ServeArchiveType::ServeArchiveTreeRequest_ArchiveType_ZIP : ServeArchiveType::ServeArchiveTreeRequest_ArchiveType_TAR; } auto PragmaSpecialToSymlinksResolve( std::optional const& resolve_symlinks) -> justbuild::just_serve::ServeArchiveTreeRequest_SymlinksResolve { using ServeSymlinksResolve = justbuild::just_serve::ServeArchiveTreeRequest_SymlinksResolve; if (not resolve_symlinks) { return ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_NONE; } switch (resolve_symlinks.value()) { case PragmaSpecial::Ignore: { return ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_IGNORE; } case PragmaSpecial::ResolvePartially: { return ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_PARTIAL; } case PragmaSpecial::ResolveCompletely: { return ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_COMPLETE; } } // default return, to avoid [-Werror=return-type] error return ServeSymlinksResolve::ServeArchiveTreeRequest_SymlinksResolve_NONE; } } // namespace SourceTreeClient::SourceTreeClient( ServerAddress const& address, HashFunction hash_function, gsl::not_null const& remote_context) noexcept : hash_function_{hash_function} { stub_ = justbuild::just_serve::SourceTree::NewStub(CreateChannelWithCredentials( address.host, address.port, remote_context->auth)); } auto SourceTreeClient::ServeCommitTree(std::string const& commit_id, std::string const& subdir, bool sync_tree) const noexcept -> result_t { justbuild::just_serve::ServeCommitTreeRequest request{}; request.set_commit(commit_id); request.set_subdir(subdir); request.set_sync_tree(sync_tree); grpc::ClientContext context; justbuild::just_serve::ServeCommitTreeResponse response; grpc::Status status = stub_->ServeCommitTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeCommitTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeCommitTree response returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeCommitTreeResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } TreeResult result = {response.tree(), std::nullopt}; // if asked to sync, get digest from response if (sync_tree) { auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), response.digest()); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } result.digest = *std::move(digest); } return result; // success } auto SourceTreeClient::ServeArchiveTree( std::string const& content, std::string const& archive_type, std::string const& subdir, std::optional const& resolve_symlinks, bool sync_tree) const noexcept -> result_t { justbuild::just_serve::ServeArchiveTreeRequest request{}; request.set_content(content); request.set_archive_type(StringToArchiveType(archive_type)); request.set_subdir(subdir); request.set_resolve_symlinks( PragmaSpecialToSymlinksResolve(resolve_symlinks)); request.set_sync_tree(sync_tree); grpc::ClientContext context; justbuild::just_serve::ServeArchiveTreeResponse response; grpc::Status status = stub_->ServeArchiveTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeArchiveTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeArchiveTree response returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeArchiveTreeResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } TreeResult result = {response.tree(), std::nullopt}; // if asked to sync, get digest from response if (sync_tree) { auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), response.digest()); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } result.digest = *std::move(digest); } return result; // success } auto SourceTreeClient::ServeDistdirTree( std::shared_ptr> const& distfiles, bool sync_tree) const noexcept -> result_t { justbuild::just_serve::ServeDistdirTreeRequest request{}; for (auto const& [k, v] : *distfiles) { auto* distfile = request.add_distfiles(); distfile->set_name(k); distfile->set_content(v); distfile->set_executable(false); } request.set_sync_tree(sync_tree); grpc::ClientContext context; justbuild::just_serve::ServeDistdirTreeResponse response; grpc::Status status = stub_->ServeDistdirTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeDistdirTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeDistdirTree response returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeDistdirTreeResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } TreeResult result = {response.tree(), std::nullopt}; // if asked to sync, get digest from response if (sync_tree) { auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), response.digest()); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } result.digest = *std::move(digest); } return result; // success } auto SourceTreeClient::ServeForeignFileTree(const std::string& content, const std::string& name, bool executable) const noexcept -> result_t { justbuild::just_serve::ServeDistdirTreeRequest request{}; auto* distfile = request.add_distfiles(); distfile->set_name(name); distfile->set_content(content); distfile->set_executable(executable); grpc::ClientContext context; justbuild::just_serve::ServeDistdirTreeResponse response; grpc::Status status = stub_->ServeDistdirTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeDistdirTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeDistdirTree called for foreign file response " "returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeDistdirTreeResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } return TreeResult{response.tree(), std::nullopt}; // success } auto SourceTreeClient::ServeContent(std::string const& content) const noexcept -> expected { justbuild::just_serve::ServeContentRequest request{}; request.set_content(content); grpc::ClientContext context; justbuild::just_serve::ServeContentResponse response; grpc::Status status = stub_->ServeContent(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeContentResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeContent response returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeContentResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), response.digest()); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } return *std::move(digest); // success } auto SourceTreeClient::ServeTree(std::string const& tree_id) const noexcept -> expected { justbuild::just_serve::ServeTreeRequest request{}; request.set_tree(tree_id); grpc::ClientContext context; justbuild::just_serve::ServeTreeResponse response; grpc::Status status = stub_->ServeTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ServeTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "ServeTree response returned with {}", static_cast(response.status())); return unexpected{ response.status() != ::justbuild::just_serve::ServeTreeResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), response.digest()); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } return *std::move(digest); // success } auto SourceTreeClient::CheckRootTree(std::string const& tree_id) const noexcept -> std::optional { justbuild::just_serve::CheckRootTreeRequest request{}; request.set_tree(tree_id); grpc::ClientContext context; justbuild::just_serve::CheckRootTreeResponse response; grpc::Status status = stub_->CheckRootTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return std::nullopt; } if (response.status() != ::justbuild::just_serve::CheckRootTreeResponse::OK) { if (response.status() != ::justbuild::just_serve::CheckRootTreeResponse::NOT_FOUND) { logger_.Emit(LogLevel::Debug, "CheckRootTree response returned with {}", static_cast(response.status())); return std::nullopt; // tree lookup failed } return false; // tree not found } return true; // tree found } auto SourceTreeClient::GetRemoteTree( ArtifactDigest const& digest) const noexcept -> bool { justbuild::just_serve::GetRemoteTreeRequest request{}; (*request.mutable_digest()) = ArtifactDigestFactory::ToBazel(digest); grpc::ClientContext context; justbuild::just_serve::GetRemoteTreeResponse response; grpc::Status status = stub_->GetRemoteTree(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return false; } if (response.status() != ::justbuild::just_serve::GetRemoteTreeResponse::OK) { logger_.Emit(LogLevel::Debug, "GetRemoteTree response returned with {}", static_cast(response.status())); return false; // retrieving tree or import-to-git failed } // success! return true; } auto SourceTreeClient::ComputeTreeStructure(ArtifactDigest const& tree) const noexcept -> expected { justbuild::just_serve::ComputeTreeStructureRequest request{}; request.set_tree(tree.hash()); grpc::ClientContext context; justbuild::just_serve::ComputeTreeStructureResponse response; grpc::Status status = stub_->ComputeTreeStructure(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Debug, status); return unexpected{GitLookupError::Fatal}; } if (response.status() != ::justbuild::just_serve::ComputeTreeStructureResponse::OK) { logger_.Emit(LogLevel::Debug, "ComputeTreeStructure response returned with {}", static_cast(response.status())); return unexpected{response.status() != ::justbuild::just_serve:: ComputeTreeStructureResponse::NOT_FOUND ? GitLookupError::Fatal : GitLookupError::NotFound}; } // ComputeTreeStructure operates on git digests. auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, response.tree_structure_hash(), /*size_unknown=*/0, /*is_tree=*/true); if (not digest) { logger_.Emit(LogLevel::Debug, std::move(digest).error()); return unexpected{GitLookupError::Fatal}; } return *std::move(digest); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/source_tree_client.hpp000066400000000000000000000160601516554100600322440ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_SOURCE_TREE_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_SOURCE_TREE_CLIENT_HPP #include #include #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" /// Implements client side for SourceTree service defined in: /// src/buildtool/serve_api/serve_service/just_serve.proto class SourceTreeClient { public: explicit SourceTreeClient( ServerAddress const& address, HashFunction hash_function, gsl::not_null const& remote_context) noexcept; struct TreeResult { std::string tree; std::optional digest; }; using result_t = expected; /// \brief Retrieve the Git tree of a given commit, if known by the /// endpoint. It is a fatal error if the commit is known to the endpoint but /// no result is able to be returned. /// \param[in] commit_id Hash of the Git commit to look up. /// \param[in] subdir Relative path of the tree inside commit. /// \param[in] sync_tree Sync tree to the remote-execution endpoint. /// \returns The optional tree digest on success or an unexpected error /// (fatal or commit or subtree not found). [[nodiscard]] auto ServeCommitTree(std::string const& commit_id, std::string const& subdir, bool sync_tree) const noexcept -> result_t; /// \brief Retrieve the Git tree of an archive content, if known by the /// endpoint. It is a fatal error if the content blob is known to the /// endpoint but no result is able to be returned. /// \param[in] content Hash of the archive content to look up. /// \param[in] archive_type Type of archive ("archive"|"zip"). /// \param[in] subdir Relative path of the tree inside archive. /// \param[in] resolve_symlinks Optional enum to state how symlinks in the /// archive should be handled if the tree has to be actually computed. /// \param[in] sync_tree Sync tree to the remote-execution endpoint. /// \returns The optional tree digest on success or an unexpected error /// (fatal or content blob not found). [[nodiscard]] auto ServeArchiveTree( std::string const& content, std::string const& archive_type, std::string const& subdir, std::optional const& resolve_symlinks, bool sync_tree) const noexcept -> result_t; /// \brief Retrieve the Git tree of a directory of distfiles, if all the /// content blobs are known by the endpoint. It is a fatal error if all /// content blobs are known but no result is able to be returned. /// \param[in] distfiles Mapping from distfile names to content blob ids. /// \param[in] sync_tree Sync tree and all distfile blobs to the /// remote-execution endpoint. /// \returns The optional tree digest on success or an unexpected error /// (fatal or at least one distfile blob missing). [[nodiscard]] auto ServeDistdirTree( std::shared_ptr> const& distfiles, bool sync_tree) const noexcept -> result_t; /// \brief Retrieve the Git tree of a foreign-file directory, if all content /// blobs are known to the endpoint and, as a side-effect, make that tree /// known to the serve endpoint. /// \param[in] content Hash of the foreign-file content. /// \param[in] name Name of the foreign-file. /// \param[in] executable Executable flag of foreign-file. /// \returns The optional tree digest on success or an unexpected error /// (fatal or content not found). [[nodiscard]] auto ServeForeignFileTree(const std::string& content, const std::string& name, bool executable) const noexcept -> result_t; /// \brief Make a given content blob available in remote CAS, if known by /// serve remote. /// \param[in] content Hash of the archive content to look up. /// \returns Flag to state whether content is in remote CAS. [[nodiscard]] auto ServeContent(std::string const& content) const noexcept -> expected; /// \brief Make a given tree available in remote CAS, if known by serve /// remote. /// \param[in] tree_id Identifier of the Git tree to look up. /// \returns Flag to state whether tree is in remote CAS. [[nodiscard]] auto ServeTree(std::string const& tree_id) const noexcept -> expected; /// \brief Checks if the serve endpoint has a given tree locally available /// and makes it available for a serve-orchestrated build. /// \param[in] tree_id Identifier of the Git tree to look up. /// \returns Flag to state whether tree is known or not, or nullopt on /// errors. [[nodiscard]] auto CheckRootTree(std::string const& tree_id) const noexcept -> std::optional; /// \brief Retrieve tree from the CAS of the associated remote-execution /// endpoint and makes it available for a serve-orchestrated build. /// \param[in] digest Tree to retrieve. /// \returns Flag to state whether tree was successfully imported into the /// local Git storage or not. [[nodiscard]] auto GetRemoteTree( ArtifactDigest const& digest) const noexcept -> bool; /// \brief Compute tree structure of a tree /// \param tree Tree to compute /// \return Git digest of the tree structure computed for tree on /// success or an unexpected error on failure (fatal or content not found). [[nodiscard]] auto ComputeTreeStructure(ArtifactDigest const& tree) const noexcept -> expected; private: HashFunction hash_function_; // hash function of the remote std::unique_ptr stub_; Logger logger_{"RemoteSourceTreeClient"}; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_SOURCE_TREE_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/target_client.cpp000066400000000000000000000257031516554100600312120ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/remote/target_client.hpp" #include #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "justbuild/just_serve/just_serve.pb.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/expected.hpp" namespace { [[nodiscard]] auto GetTargetValue( HashFunction::Type hash_type, justbuild::just_serve::ServeTargetResponse const& response) noexcept -> std::optional { if (not response.has_target_value()) { return std::nullopt; } auto result = ArtifactDigestFactory::FromBazel(hash_type, response.target_value()); if (not result) { return std::nullopt; } return *std::move(result); } } // namespace TargetClient::TargetClient( ServerAddress const& address, gsl::not_null const& storage, gsl::not_null const& remote_context, gsl::not_null const& apis) noexcept : storage_{*storage}, exec_config_{*remote_context->exec_config}, apis_{*apis} { stub_ = justbuild::just_serve::Target::NewStub(CreateChannelWithCredentials( address.host, address.port, remote_context->auth)); } auto TargetClient::ServeTarget(const TargetCacheKey& key, const ArtifactDigest& repo_key, bool keep_artifact_root) const noexcept -> std::optional { // make sure the blob containing the key is in the remote cas if (not apis_.local->RetrieveToCas({key.Id()}, *apis_.remote)) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Failed to retrieve to remote cas ObjectInfo {}", key.Id().ToString())}; } // make sure the repository configuration blob is in the remote cas if (not apis_.local->RetrieveToCas( {Artifact::ObjectInfo{.digest = repo_key, .type = ObjectType::File}}, *apis_.remote)) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Failed to retrieve to remote cas blob {}", repo_key.hash())}; } // add target cache key to request justbuild::just_serve::ServeTargetRequest request{}; *request.mutable_target_cache_key_id() = ArtifactDigestFactory::ToBazel(key.Id().digest); request.set_keep_artifact_root(keep_artifact_root); // add execution properties to request for (auto const& [k, v] : exec_config_.platform_properties) { auto* prop = request.add_execution_properties(); prop->set_name(k); prop->set_value(v); } // add dispatch information to request, while ensuring blob is uploaded // to remote cas std::optional dispatch_digest; try { auto dispatch_list = nlohmann::json::array(); for (auto const& [props, endpoint] : exec_config_.dispatch) { auto entry = nlohmann::json::array(); entry.push_back(nlohmann::json(props)); entry.push_back(endpoint.ToJson()); dispatch_list.push_back(entry); } dispatch_digest = storage_.CAS().StoreBlob(dispatch_list.dump(2)); } catch (std::exception const& ex) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Populating dispatch JSON array failed with:\n{}", ex.what())}; } if (not dispatch_digest) { return serve_target_result_t{ std::in_place_index<1>, "Failed to add dispatch info to local cas"}; } auto const dispatch_info = Artifact::ObjectInfo{.digest = *dispatch_digest, .type = ObjectType::File}; if (not apis_.local->RetrieveToCas({dispatch_info}, *apis_.remote)) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Failed to upload blob {} to remote cas", dispatch_info.ToString())}; } (*request.mutable_dispatch_info()) = ArtifactDigestFactory::ToBazel(*dispatch_digest); request.mutable_required_digests()->Add( ArtifactDigestFactory::ToBazel(repo_key)); // call rpc grpc::ClientContext context; justbuild::just_serve::ServeTargetResponse response; auto const& status = stub_->ServeTarget(&context, request, &response); // differentiate status codes switch (status.error_code()) { case grpc::StatusCode::OK: { // if log has been set, pass it along as index 0 if (response.has_log()) { auto log_digest = ArtifactDigestFactory::FromBazel( storage_.GetHashFunction().GetType(), response.log()); if (not log_digest) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Failed to convert log digest: {}", std::move(log_digest).error())}; } return serve_target_result_t{ std::in_place_index<0>, Artifact::ObjectInfo{*log_digest, ObjectType::File, false} .ToString()}; } // if no log has been set, it must have the target cache value auto const target_value_dgst = GetTargetValue(storage_.GetHashFunction().GetType(), response); if (not target_value_dgst) { return serve_target_result_t{ std::in_place_index<1>, "Serve endpoint failed to set expected response field"}; } auto const obj_info = Artifact::ObjectInfo{ .digest = *target_value_dgst, .type = ObjectType::File}; if (not apis_.local->IsAvailable(*target_value_dgst)) { if (not apis_.remote->RetrieveToCas({obj_info}, *apis_.local)) { return serve_target_result_t{ std::in_place_index<1>, fmt::format( "Failed to retrieve blob {} from remote cas", obj_info.ToString())}; } } auto const& target_value_str = apis_.local->RetrieveToMemory(obj_info); if (not target_value_str) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Failed to retrieve blob {} from local cas", obj_info.ToString())}; } try { auto const result = TargetCacheEntry::FromJson( storage_.GetHashFunction().GetType(), nlohmann::json::parse(*target_value_str)); // return the target cache value information return serve_target_result_t{std::in_place_index<3>, std::make_pair(result, obj_info)}; } catch (std::exception const& ex) { return serve_target_result_t{ std::in_place_index<1>, fmt::format("Parsing target cache value failed with:\n{}", ex.what())}; } } case grpc::StatusCode::INTERNAL: { return serve_target_result_t{ std::in_place_index<1>, fmt::format( "Serve endpoint reported the fatal internal error:\n{}", status.error_message())}; } case grpc::StatusCode::NOT_FOUND: { return std::nullopt; // this might allow a local build to continue } default:; // fallthrough } return serve_target_result_t{ std::in_place_index<2>, fmt::format("Serve endpoint failed with:\n{}", status.error_message())}; } auto TargetClient::ServeTargetVariables(std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional> { justbuild::just_serve::ServeTargetVariablesRequest request{}; request.set_root_tree(target_root_id); request.set_target_file(target_file); request.set_target(target); grpc::ClientContext context; justbuild::just_serve::ServeTargetVariablesResponse response; grpc::Status status = stub_->ServeTargetVariables(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Error, status); return std::nullopt; } auto size = response.flexible_config_size(); if (size == 0) { return std::vector(); } std::vector res{}; res.reserve(size); for (auto const& var : response.flexible_config()) { res.emplace_back(var); } return res; } auto TargetClient::ServeTargetDescription( std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional { justbuild::just_serve::ServeTargetDescriptionRequest request{}; request.set_root_tree(target_root_id); request.set_target_file(target_file); request.set_target(target); grpc::ClientContext context; justbuild::just_serve::ServeTargetDescriptionResponse response; grpc::Status status = stub_->ServeTargetDescription(&context, request, &response); if (not status.ok()) { LogStatus(&logger_, LogLevel::Error, status); return std::nullopt; } auto result = ArtifactDigestFactory::FromBazel( storage_.GetHashFunction().GetType(), response.description_id()); if (not result) { logger_.Emit(LogLevel::Error, "{}", std::move(result).error()); return std::nullopt; } return *std::move(result); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/remote/target_client.hpp000066400000000000000000000113671516554100600312200ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" /// \brief Result union for the ServeTarget request. /// Index 0 will contain the digest of the blob containing the logged /// analysis/build failure received from the endpoint, as a string; this should /// also trigger a local build fail. /// Index 1 will contain the message of any INTERNAL error on the endpoint, as /// a string; this should trigger a local build fail. /// Index 2 will contain any other failure message, as a string; local builds /// might be able to continue, but with a warning. /// Index 3 will contain the target cache value information on success. using serve_target_result_t = std::variant>; /// Implements client side for Target service defined in: /// src/buildtool/serve_api/serve_service/just_serve.proto class TargetClient { public: explicit TargetClient( ServerAddress const& address, gsl::not_null const& storage, gsl::not_null const& remote_context, gsl::not_null const& apis) noexcept; /// \brief Retrieve the pair of TargetCacheEntry and ObjectInfo associated /// to the given key. /// \param[in] key The TargetCacheKey of an export target. /// \param[in] repo_key The RepositoryKey to upload as precondition. /// \returns A correspondingly populated result union, or nullopt if remote /// reported that the target was not found. [[nodiscard]] auto ServeTarget(const TargetCacheKey& key, const ArtifactDigest& repo_key, bool keep_artifact_root = false) const noexcept -> std::optional; /// \brief Retrieve the flexible config variables of an export target. /// \param[in] target_root_id Hash of target-level root tree. /// \param[in] target_file Relative path of the target file. /// \param[in] target Name of the target to interrogate. /// \returns The list of flexible config variables, or nullopt on errors. [[nodiscard]] auto ServeTargetVariables(std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional>; /// \brief Retrieve the artifact digest of the blob containing the export /// target description fields required by just describe. /// \param[in] target_root_id Hash of target-level root tree. /// \param[in] target_file Relative path of the target file. /// \param[in] target Name of the target to interrogate. /// \returns The artifact digest, or nullopt on errors. [[nodiscard]] auto ServeTargetDescription(std::string const& target_root_id, std::string const& target_file, std::string const& target) const noexcept -> std::optional; private: Storage const& storage_; RemoteExecutionConfig const& exec_config_; ApiBundle const& apis_; std::unique_ptr stub_; Logger logger_{"RemoteTargetClient"}; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/000077500000000000000000000000001516554100600272245ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/TARGETS000066400000000000000000000204431516554100600302630ustar00rootroot00000000000000{ "just_serve_proto": { "type": ["@", "rules", "proto", "library"] , "name": ["just_serve_proto"] , "service": ["yes"] , "srcs": ["just_serve.proto"] , "deps": [["@", "bazel_remote_apis", "", "remote_execution_proto"]] , "stage": ["justbuild", "just_serve"] } , "source_tree": { "type": ["@", "rules", "CC", "library"] , "name": ["source_tree"] , "hdrs": ["source_tree.hpp"] , "srcs": ["source_tree.cpp"] , "proto": ["just_serve_proto"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/file_system/symlinks", "resolve_symlinks_map"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/serve_api/remote", "config"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "garbage_collector"] , ["src/buildtool/storage", "repository_garbage_collector"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/tree_structure", "tree_structure_utils"] , ["src/utils/archive", "archive_ops"] , ["src/utils/cpp", "file_locking"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "tmp_dir"] ] } , "serve_server_implementation": { "type": ["@", "rules", "CC", "library"] , "name": ["serve_server_implementation"] , "hdrs": ["serve_server_implementation.hpp"] , "srcs": ["serve_server_implementation.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ "configuration_service" , "source_tree" , "target_service" , ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common/remote", "port"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/execution_service", "ac_server"] , ["src/buildtool/execution_api/execution_service", "bytestream_server"] , ["src/buildtool/execution_api/execution_service", "capabilities_server"] , ["src/buildtool/execution_api/execution_service", "cas_server"] , ["src/buildtool/execution_api/execution_service", "execution_server"] , ["src/buildtool/execution_api/execution_service", "operations_server"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/file_system", "atomic"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "type_safe_arithmetic"] ] } , "target_service": { "type": ["@", "rules", "CC", "library"] , "name": ["target_service"] , "hdrs": ["target.hpp"] , "srcs": ["target.cpp"] , "proto": ["just_serve_proto"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ "target_utils" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["@", "protoc", "", "libprotobuf"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "statistics"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_engine/executor", "context"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/main", "analyse"] , ["src/buildtool/main", "analyse_context"] , ["src/buildtool/main", "build_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "progress_reporter"] , ["src/buildtool/storage", "backend_description"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "garbage_collector"] , ["src/buildtool/storage", "repository_garbage_collector"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "tmp_dir"] ] } , "configuration_service": { "type": ["@", "rules", "CC", "library"] , "name": ["configuration_service"] , "hdrs": ["configuration.hpp"] , "srcs": ["configuration.cpp"] , "proto": ["just_serve_proto"] , "deps": [ ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/serve_api/remote", "config"] ] , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/logging", "log_level"] ] } , "target_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["target_utils"] , "hdrs": ["target_utils.hpp"] , "srcs": ["target_utils.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "config"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/logging", "log_level"] , ["src/utils/cpp", "expected"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/configuration.cpp000066400000000000000000000040671516554100600326060ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/serve_service/configuration.hpp" #include #include #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/logging/log_level.hpp" auto ConfigurationService::RemoteExecutionEndpoint( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::RemoteExecutionEndpointRequest* /*request*/, ::justbuild::just_serve::RemoteExecutionEndpointResponse* response) -> ::grpc::Status { logger_->Emit(LogLevel::Debug, "RemoteExecutionEndpoint()"); std::optional address; if (serve_config_.client_execution_address.has_value()) { address = serve_config_.client_execution_address; } else { address = remote_config_.remote_address; } response->set_address(address ? address->ToJson().dump() : std::string()); response->set_remote_instance_name(remote_config_.remote_instance_name); return ::grpc::Status::OK; } auto ConfigurationService::Compatibility( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::CompatibilityRequest* /*request*/, ::justbuild::just_serve::CompatibilityResponse* response) -> ::grpc::Status { logger_->Emit(LogLevel::Debug, "Compatibility()"); response->set_compatible(not ProtocolTraits::IsNative(hash_type_)); return ::grpc::Status::OK; } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/configuration.hpp000066400000000000000000000055201516554100600326060ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP #define INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "justbuild/just_serve/just_serve.pb.h" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" // This service can be used by the client to double-check the server // configuration. class ConfigurationService final : public justbuild::just_serve::Configuration::Service { public: explicit ConfigurationService( HashFunction::Type hash_type, gsl::not_null const& remote_config, gsl::not_null const& serve_config) noexcept : hash_type_{hash_type}, remote_config_{*remote_config}, serve_config_{*serve_config} {} // Returns the address of the associated remote endpoint, if set, // or an empty string signaling that the serve endpoint acts also // as a remote execution endpoint. // // There are no method-specific errors. auto RemoteExecutionEndpoint( ::grpc::ServerContext* context, const ::justbuild::just_serve::RemoteExecutionEndpointRequest* request, ::justbuild::just_serve::RemoteExecutionEndpointResponse* response) -> ::grpc::Status override; // Returns a flag signaling whether the associated remote-execution // endpoint uses the standard remote-execution protocol. // // There are no method-specific errors. auto Compatibility( ::grpc::ServerContext* context, const ::justbuild::just_serve::CompatibilityRequest* request, ::justbuild::just_serve::CompatibilityResponse* response) -> ::grpc::Status override; private: HashFunction::Type hash_type_; RemoteExecutionConfig const& remote_config_; RemoteServeConfig const& serve_config_; std::shared_ptr logger_{ std::make_shared("configuration-service")}; }; #endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/just_serve.proto000066400000000000000000000476141516554100600325160ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package justbuild.just_serve; import "build/bazel/remote/execution/v2/remote_execution.proto"; // A request message for // [SourceTree.ServeCommitTree][justbuild.just_serve.SourceTree.ServeCommitTree]. message ServeCommitTreeRequest { // The Git commit identifier to be searched on the server. string commit = 1; // Relative path of requested tree with respect to the commit root. string subdir = 2; // If set to true and the tree is found, it will be uploaded to the // remote-execution endpoint. bool sync_tree = 3; } // A response message for // [SourceTree.ServeCommitTree][justbuild.just_serve.SourceTree.ServeCommitTree]. message ServeCommitTreeResponse { // The requested Git tree identifier. string tree = 1; enum ServeCommitTreeStatus { // All good. OK = 0; // Failed to upload tree remotely. SYNC_ERROR = 1; // Commit not found. NOT_FOUND = 2; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 3; } // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeCommitTreeStatus status = 2; // The digest of the requested tree, which can be used to retrieve it from // the associated remote-execution endpoint CAS, if tree was uploaded to the // remote-execution endpoint. build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for // [SourceTree.ServeArchiveTree][justbuild.just_serve.SourceTree.ServeArchiveTree]. message ServeArchiveTreeRequest { // The Git blob identifier of the archive. string content = 1; enum ArchiveType { TAR = 0; ZIP = 1; } // The type of archive this blob should be treated as. ArchiveType archive_type = 2; // Relative path of requested tree with respect to the archive root. string subdir = 3; enum SymlinksResolve { // Leave all entries as they are. NONE = 0; // Ignore all symlinks. IGNORE = 1; // Resolve only upwards symlinks. PARTIAL = 2; // Resolve all symlinks. COMPLETE = 3; } // How symlinks inside the archive directory tree should be handled. SymlinksResolve resolve_symlinks = 4; // If set to true and the tree is found, it will be uploaded to the // remote-execution endpoint. bool sync_tree = 5; } // A response message for // [SourceTree.ServeArchiveTree][justbuild.just_serve.SourceTree.ServeArchiveTree]. message ServeArchiveTreeResponse { // The requested Git tree identifier. string tree = 1; enum ServeArchiveTreeStatus{ // All good. OK = 0; // Failed to upload tree remotely. SYNC_ERROR = 1; // Failed to unpack as archive of the specified type. UNPACK_ERROR = 2; // Failed to resolve symlinks as requested. RESOLVE_ERROR = 3; // Content blob not found. NOT_FOUND = 4; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 5; } // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeArchiveTreeStatus status = 2; // The digest of the requested tree, which can be used to retrieve it from // the associated remote-execution endpoint CAS, if tree was uploaded to the // remote-execution endpoint. build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for // [SourceTree.ServeDistdirTree][justbuild.just_serve.SourceTree.ServeDistdirTree]. message ServeDistdirTreeRequest { // A distfile item. message Distfile { // The name of the distfile. string name = 1; // The Git blob identifier of the distfile content. string content = 2; // Whether the blob should occur executable in the resulting // directory. bool executable = 3; } // The list of distfiles. repeated Distfile distfiles = 1; // If set to true and all distfile blobs are found, the resulting tree // and all the content blobs will be uploaded to the remote-execution // endpoint. bool sync_tree = 2; } // A response message for // [SourceTree.ServeDistdirTree][justbuild.just_serve.SourceTree.ServeDistdirTree]. message ServeDistdirTreeResponse { // The requested Git tree identifier. string tree = 1; enum ServeDistdirTreeStatus{ // All good OK = 0; // Failed to upload content blobs remotely SYNC_ERROR = 1; // At least one content blob is missing NOT_FOUND = 2; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 3; } // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeDistdirTreeStatus status = 2; // The digest of the requested tree, which can be used to retrieve it from // the associated remote-execution endpoint CAS, if tree was uploaded to the // remote-execution endpoint. build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for // [SourceTree.ServeContent][justbuild.just_serve.SourceTree.ServeContent]. message ServeContentRequest { // The content Git blob identifier. string content = 1; } // A response message for // [SourceTree.ServeContent][justbuild.just_serve.SourceTree.ServeContent]. message ServeContentResponse { enum ServeContentStatus{ // All good. OK = 0; // Content blob not known. NOT_FOUND = 1; // Failed to upload content blob to remote CAS. SYNC_ERROR = 2; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 3; } // If the status has a code `OK`, the content blob is in the remote CAS. ServeContentStatus status = 1; // The digest of the requested blob, which can be used to retrieve it from // the associated remote-execution endpoint CAS. build.bazel.remote.execution.v2.Digest digest = 2; } // A request message for // [SourceTree.ServeTree][justbuild.just_serve.SourceTree.ServeTree]. message ServeTreeRequest { // The Git tree identifier. string tree = 1; } // A response message for // [SourceTree.ServeTree][justbuild.just_serve.SourceTree.ServeTree]. message ServeTreeResponse { enum ServeTreeStatus{ // All good. OK = 0; // Tree not known. NOT_FOUND = 1; // Failed to upload tree to remote CAS. SYNC_ERROR = 2; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 3; } // If the status has a code `OK`, the tree is in the remote CAS. ServeTreeStatus status = 1; // The digest of the requested tree, which can be used to retrieve it from // the associated remote-execution endpoint CAS. build.bazel.remote.execution.v2.Digest digest = 2; } // A request message for // [SourceTree.CheckRootTree][justbuild.just_serve.SourceTree.CheckRootTree]. message CheckRootTreeRequest { // The Git tree identifier. string tree = 1; } // A response message for // [SourceTree.CheckRootTree][justbuild.just_serve.SourceTree.CheckRootTree]. message CheckRootTreeResponse { enum CheckRootTreeStatus{ // All good. OK = 0; // Tree not known. NOT_FOUND = 1; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 2; } // If the status has a code `OK`, the tree is known locally and available // in a location where this serve instance can build against. CheckRootTreeStatus status = 1; } // A request message for // [SourceTree.GetRemoteTree][justbuild.just_serve.SourceTree.GetRemoteTree]. message GetRemoteTreeRequest { reserved 1; // The Git tree identifier in an earlier version of the API. // The tree digest, which can be used to retrieve it from the associated // remote-execution endpoint. build.bazel.remote.execution.v2.Digest digest = 2; } // A response message for // [SourceTree.GetRemoteTree][justbuild.just_serve.SourceTree.GetRemoteTree]. message GetRemoteTreeResponse { enum GetRemoteTreeStatus{ // All good. OK = 0; // Tree is not found in remote CAS or it failed to be retrieved. FAILED_PRECONDITION = 1; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 2; } // If the status has a code `OK`, the tree is available locally in a // location where this serve instance can build against. GetRemoteTreeStatus status = 1; } // A request message for // [SourceTree.ComputeTreeStructure][justbuild.just_serve.SourceTree.ComputeTreeStructure]. message ComputeTreeStructureRequest { // The Git tree identifier of the tree to compute the tree structure for. string tree = 1; } // A response message for // [SourceTree.ComputeTreeStructure][justbuild.just_serve.SourceTree.ComputeTreeStructure]. message ComputeTreeStructureResponse { enum ComputeTreeStructureStatus{ // All good. OK = 0; // Tree not known. NOT_FOUND = 1; // An internal error occurred during the resolution of the request. INTERNAL_ERROR = 2; } // If the status has a code `OK`, the tree structure is available locally in // a location where this serve instance can build against. ComputeTreeStructureStatus status = 1; // The Git tree identifier of the tree structure. string tree_structure_hash = 2; } // Service for improved interaction with the target-level cache. service SourceTree { // Retrieve the Git-subtree identifier from a given Git commit. // // There are no method-specific errors. rpc ServeCommitTree(ServeCommitTreeRequest) returns (ServeCommitTreeResponse) {} // Retrieve the Git-subtree identifier for the tree obtained // by unpacking an archive with a given blob identifier. // // There are no method-specific errors. rpc ServeArchiveTree(ServeArchiveTreeRequest) returns (ServeArchiveTreeResponse) {} // Compute the Git-tree identifier for the tree containing the content // blobs of a list of distfiles. The implementation must only return the // tree identifier if ALL content blobs are known. // // There are no method-specific errors. rpc ServeDistdirTree(ServeDistdirTreeRequest) returns (ServeDistdirTreeResponse) {} // Make a given content blob available in remote CAS, if blob is known. // // There are no method-specific errors. rpc ServeContent(ServeContentRequest) returns (ServeContentResponse) {} // Make a given tree available in remote CAS, if tree is known. // // There are no method-specific errors. rpc ServeTree(ServeTreeRequest) returns (ServeTreeResponse) {} // Check if a Git-tree is locally known and, if found, make it available // in a location where this serve instance can build against. // The implementation should not interrogate the associated remote-execution // endpoint at any point during the completion of this request. // // There are no method-specific errors. rpc CheckRootTree(CheckRootTreeRequest) returns (CheckRootTreeResponse) {} // Retrieve a given tree from the CAS of the associated remote-execution // endpoint and make it available in a location where this serve instance // can build against. // // There are no method-specific errors. rpc GetRemoteTree(GetRemoteTreeRequest) returns (GetRemoteTreeResponse) {} // Compute the tree structure of the given tree and return the Git tree // identifier of the resulting structure. // // There are no method-specific errors. rpc ComputeTreeStructure(ComputeTreeStructureRequest) returns (ComputeTreeStructureResponse) {} } // A request message for // [Target.ServeTarget][justbuild.just_serve.Target.ServeTarget]. message ServeTargetRequest { // Digest of the blob containing the target description. // // The client has to guarantee that the blob has been uploaded to the // remote CAS. build.bazel.remote.execution.v2.Digest target_cache_key_id = 1; // Request that serve keep the artifact stage as an additional root, // so that susequent requests to serve a target can use this root. bool keep_artifact_root = 4; // A single property of the remote-execution environment. message Property { // Property name. string name = 1; // Property value. string value = 2; } // The execution properties. The key-value pairs accumulate, with latest // value for each key being taken. repeated Property execution_properties = 2; // Digest of the blob containing the endpoint configuration. // // The client has to guarantee that the blob has been uploaded to the // remote CAS. build.bazel.remote.execution.v2.Digest dispatch_info = 3; // Digests that the serve SHOULD make available before handling the request. // Serve MUST attempt to perform execution even if it fails to receive some // digests. repeated build.bazel.remote.execution.v2.Digest required_digests = 5; } // A response message for // [Target.ServeTarget][justbuild.just_serve.Target.ServeTarget]. message ServeTargetResponse { // Digest of the blob with the JSON object containing the target-cache // value on success. The implementation must guarantee that all the // referenced objects are present in the remote CAS. build.bazel.remote.execution.v2.Digest target_value = 1; // Digest of the blob containing a log report relevant to the user, e.g., // the error report for a failed analysis or build of a known target. // The implementation must guarantee that this blob is present in the // remote CAS. build.bazel.remote.execution.v2.Digest log = 2; } // A request message for // [Target.ServeTargetVariables][justbuild.just_serve.Target.ServeTargetVariables]. message ServeTargetVariablesRequest { // Git identifier of the target-level root tree. string root_tree = 1; // Relative path of the targets file inside the root tree. string target_file = 2; // Name of the export target to look up. string target = 3; } // A response message for // [Target.ServeTargetVariables][justbuild.just_serve.Target.ServeTargetVariables]. message ServeTargetVariablesResponse { // List of flexible configuration variables. repeated string flexible_config = 1; } // A request message for // [Target.ServeTargetDescription][justbuild.just_serve.Target.ServeTargetDescription]. message ServeTargetDescriptionRequest { // Git identifier of the target-level root tree. string root_tree = 1; // Relative path of the targets file inside the root tree. string target_file = 2; // Name of the export target to look up. string target = 3; } // A response message for // [Target.ServeTargetDescription][justbuild.just_serve.Target.ServeTargetDescription]. message ServeTargetDescriptionResponse { // Digest of the blob containing the desired target description fields. // The implementation must guarantee the reference object is available in // the remote CAS. build.bazel.remote.execution.v2.Digest description_id = 1; } // The target-level cache service. service Target { // Given a target-level caching key, returns the computed value. In doing // so, it can build on the associated endpoint passing the // RemoteExecutionProperties contained in the ServeTargetRequest. // The execution backend description, the resulting target cache value, // and all other artifacts referenced therein MUST be made available in // the CAS of the associated remote-execution endpoint. // // A failure to analyse or build a known target (i.e., a target for which // we have all the needed information available) should NOT be reported as // an error. Instead, the failure log should be uploaded as a blob to the // CAS of the associated remote-execution endpoint and its digest provided // to the client in the response field `log`. In this case, the field // `target_value` MUST not be set. // // If the status has a code different from `OK` or `NOT_FOUND`, the // response MUST not be used. // // Errors: // * `NOT_FOUND`: Unknown target or missing needed local information. // This should only be used for non-fatal failures. // * `FAILED_PRECONDITION`: Required entries missing in the remote // execution endpoint. // * `UNAVAILABLE`: Could not communicate with the remote-execution // endpoint. // * `INVALID_ARGUMENT`: The client provided invalid arguments in request. // * `INTERNAL`: Internally, something is very broken. rpc ServeTarget(ServeTargetRequest) returns (ServeTargetResponse) {} // Given the target-level root tree and the name of an export target, // returns the list of flexible variables from that target's description. // // If the status has a code different from `OK`, the response MUST not be // used. // // Errors: // * `FAILED_PRECONDITION`: An error occurred in retrieving the // configuration of the requested target, such as missing entries // (target-root, target file, target name), unparsable target file, or // requested target not being of "type" : "export". // * `INTERNAL`: Internally, something is very broken. rpc ServeTargetVariables(ServeTargetVariablesRequest) returns (ServeTargetVariablesResponse) {} // Given the target-level root tree and the name of an export target, // returns the digest of the blob containing the flexible variables field, // as well as the documentation fields pertaining tho the target and // its configuration variables, as taken from the target's description. // This information should be enough for a client to produce locally a // full description of said target. // // The server MUST make the returned blob available in the remote CAS. // // If the status has a code different from `OK`, the response MUST not be // used. // // Errors: // * `FAILED_PRECONDITION`: An error occurred in retrieving the // configuration of the requested target, such as missing entries // (target-root, target file, target name), unparsable target file, or // requested target not being of "type" : "export". // * `UNAVAILABLE`: Could not communicate with the remote CAS. // * `INTERNAL`: Internally, something is very broken. rpc ServeTargetDescription(ServeTargetDescriptionRequest) returns (ServeTargetDescriptionResponse) {} } // A request message for // [Configuration.RemoteExecutionEndpoint][justbuild.just_serve.Configuration.RemoteExecutionEndpoint]. message RemoteExecutionEndpointRequest {} // A response message for // [Configuration.RemoteExecutionEndpoint][justbuild.just_serve.Configuration.RemoteExecutionEndpoint]. message RemoteExecutionEndpointResponse { // The remote endpoint address in HOST:PORT format, if one is set. string address = 1; // The instance_name used for the remote endpoint. string remote_instance_name = 2; } // A request message for // [Configuration.Compatibility][justbuild.just_serve.Configuration.Compatibility]. message CompatibilityRequest {} // A response message for // [Configuration.Compatibility][justbuild.just_serve.Configuration.Compatibility]. message CompatibilityResponse { // Standard remote-execution protocol compatibility flag. bool compatible = 1; } // This service can be used by the client to double-check the server // configuration. service Configuration { // Returns the address of the associated remote endpoint, if set, // or an empty string signaling that the serve endpoint acts also // as a remote-execution endpoint. // // There are no method-specific errors. rpc RemoteExecutionEndpoint(RemoteExecutionEndpointRequest) returns (RemoteExecutionEndpointResponse) {} // Returns a flag signaling whether the associated remote-execution // endpoint uses the standard remote-execution protocol. // // There are no method-specific errors. rpc Compatibility(CompatibilityRequest) returns (CompatibilityResponse) {} } serve_server_implementation.cpp000066400000000000000000000234421516554100600354750ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/serve_service/serve_server_implementation.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/execution_service/ac_server.hpp" #include "src/buildtool/execution_api/execution_service/bytestream_server.hpp" #include "src/buildtool/execution_api/execution_service/capabilities_server.hpp" #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include "src/buildtool/execution_api/execution_service/operations_server.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/atomic.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/serve_service/configuration.hpp" #include "src/buildtool/serve_api/serve_service/source_tree.hpp" #include "src/buildtool/serve_api/serve_service/target.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/type_safe_arithmetic.hpp" auto ServeServerImpl::Create(std::optional interface, std::optional port, std::optional info_file, std::optional pid_file, gsl::not_null const& lock) noexcept -> std::optional { auto server = ServeServerImpl(lock); if (interface) { server.interface_ = std::move(*interface); } if (port) { auto parsed_port = ParsePort(*port); if (parsed_port) { server.port_ = static_cast(*parsed_port); } else { Logger::Log(LogLevel::Error, "Invalid port '{}'", *port); return std::nullopt; } } if (info_file) { server.info_file_ = std::move(*info_file); } if (pid_file) { server.pid_file_ = std::move(*pid_file); } return server; } auto ServeServerImpl::Run( RemoteServeConfig const& serve_config, gsl::not_null const& local_context, gsl::not_null const& remote_context, std::optional const& serve, ApiBundle const& apis, std::optional op_exponent, bool with_execute) -> bool { // make sure the git root directory is properly initialized if (not FileSystemManager::CreateDirectory( local_context->storage_config->GitRoot())) { Logger::Log(LogLevel::Error, "Could not create directory {}. Aborting", local_context->storage_config->GitRoot().string()); return false; } if (not GitRepo::InitAndOpen(local_context->storage_config->GitRoot(), true)) { Logger::Log( LogLevel::Error, fmt::format("could not initialize bare git repository {}", local_context->storage_config->GitRoot().string())); return false; } auto const hash_type = local_context->storage_config->hash_function.GetType(); // TargetService and ConfigurationService use the default apis, which know // how to dispatch builds. TargetService ts{&serve_config, local_context, remote_context, &apis, lock_, serve ? &*serve : nullptr}; ConfigurationService cs{ hash_type, remote_context->exec_config, &serve_config}; // For the SourceTreeService we need to always have access to a native // storage. In compatible mode, this requires creating a second local // context, as the default one is compatible. std::unique_ptr secondary_storage_config = nullptr; std::unique_ptr secondary_storage = nullptr; std::unique_ptr secondary_local_context = nullptr; IExecutionApi::Ptr secondary_local_api = nullptr; auto const is_compat = not ProtocolTraits::IsNative(hash_type); if (is_compat) { auto config = StorageConfig::Builder::Rebuild(*local_context->storage_config) .SetHashType(HashFunction::Type::GitSHA1) .Build(); if (not config) { Logger::Log(LogLevel::Error, config.error()); return false; } secondary_storage_config = std::make_unique(*std::move(config)); secondary_storage = std::make_unique( Storage::Create(&*secondary_storage_config)); secondary_local_context = std::make_unique( LocalContext{.exec_config = local_context->exec_config, .storage_config = &*secondary_storage_config, .storage = &*secondary_storage}); secondary_local_api = std::make_shared(&*secondary_local_context); } // setup the overall local api, aware of compatibility IExecutionApi::Ptr mr_local_api = std::make_shared( is_compat ? &*secondary_local_context : local_context, is_compat ? &*secondary_local_api : &*apis.local, is_compat ? &*local_context : nullptr, is_compat ? &*apis.local : nullptr); // setup the apis to pass to SourceTreeService auto const mr_apis = ApiBundle{.local = mr_local_api, .remote = apis.remote}; SourceTreeService sts{ &serve_config, &mr_apis, is_compat ? &*secondary_local_context : local_context, // native_context lock_, is_compat ? &*local_context : nullptr // compat_context }; // set up the server grpc::ServerBuilder builder; builder.RegisterService(&sts); builder.RegisterService(&ts); builder.RegisterService(&cs); // the user has not given any remote-execution endpoint // so we start a "just-execute instance" on the same process [[maybe_unused]] ExecutionServiceImpl es{ local_context, dynamic_cast(&*apis.local), op_exponent}; [[maybe_unused]] ActionCacheServiceImpl ac{local_context}; [[maybe_unused]] CASServiceImpl cas{local_context}; [[maybe_unused]] BytestreamServiceImpl b{local_context}; [[maybe_unused]] CapabilitiesServiceImpl cap{hash_type}; [[maybe_unused]] OperationsServiceImpl op{&es.GetOpCache()}; if (with_execute) { builder.RegisterService(&es) .RegisterService(&ac) .RegisterService(&cas) .RegisterService(&b) .RegisterService(&cap) .RegisterService(&op); } // check authentication credentials; currently only TLS/SSL is supported std::shared_ptr creds; if (const auto* tls_auth = std::get_if(&remote_context->auth->method); tls_auth != nullptr) { auto tls_opts = grpc::SslServerCredentialsOptions{}; tls_opts.pem_root_certs = tls_auth->ca_cert; grpc::SslServerCredentialsOptions::PemKeyCertPair keycert = { tls_auth->server_key, tls_auth->server_cert}; tls_opts.pem_key_cert_pairs.emplace_back(keycert); creds = grpc::SslServerCredentials(tls_opts); } else { creds = grpc::InsecureServerCredentials(); } builder.AddListeningPort( fmt::format("{}:{}", interface_, port_), creds, &port_); auto server = builder.BuildAndStart(); if (not server) { Logger::Log(LogLevel::Error, "Could not start serve service"); return false; } auto pid = getpid(); nlohmann::json const& info = { {"interface", interface_}, {"port", port_}, {"pid", pid}}; if (not pid_file_.empty()) { if (not FileSystemAtomic::WriteFile(pid_file_, fmt::format("{}", pid))) { server->Shutdown(); return false; } } auto const& info_str = nlohmann::to_string(info); Logger::Log(LogLevel::Info, "{}serve{} service{} started: {}", ProtocolTraits::IsNative(hash_type) ? "" : "compatible ", with_execute ? " and execute" : "", with_execute ? "s" : "", info_str); if (not info_file_.empty()) { if (not FileSystemAtomic::WriteFile(info_file_, info_str)) { server->Shutdown(); return false; } } server->Wait(); return true; } #endif // BOOTSTRAP_BUILD_TOOL serve_server_implementation.hpp000066400000000000000000000054021516554100600354760ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef SERVE_SERVER_IMPLEMENTATION_HPP #define SERVE_SERVER_IMPLEMENTATION_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" class ServeServerImpl final { public: [[nodiscard]] static auto Create( std::optional interface, std::optional port, std::optional info_file, std::optional pid_file, gsl::not_null const& lock) noexcept -> std::optional; ~ServeServerImpl() noexcept = default; ServeServerImpl(ServeServerImpl const&) = delete; auto operator=(ServeServerImpl const&) -> ServeServerImpl& = delete; ServeServerImpl(ServeServerImpl&&) noexcept = default; auto operator=(ServeServerImpl&&) noexcept -> ServeServerImpl& = default; /// \brief Start the serve service. /// \param serve_config RemoteServeConfig to be used. /// \param local_context Aggregate of storage and local configs to use. /// \param serve ServeApi to be used. /// \param with_execute Flag specifying if just serve should act also as /// just execute (i.e., start remote execution services with same interface) auto Run(RemoteServeConfig const& serve_config, gsl::not_null const& local_context, gsl::not_null const& remote_context, std::optional const& serve, ApiBundle const& apis, std::optional op_exponent, bool with_execute) -> bool; private: explicit ServeServerImpl(gsl::not_null lock) noexcept : lock_{lock} {}; gsl::not_null lock_; std::string interface_{"127.0.0.1"}; int port_{0}; std::string info_file_; std::string pid_file_; }; #endif // SERVE_SERVER_IMPLEMENTATION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/source_tree.cpp000066400000000000000000002234541516554100600322610ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/serve_service/source_tree.hpp" #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/repository_garbage_collector.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/tree_structure/tree_structure_utils.hpp" #include "src/utils/archive/archive_ops.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/file_locking.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { auto ArchiveTypeToString( ::justbuild::just_serve::ServeArchiveTreeRequest_ArchiveType const& type) -> std::string { using ServeArchiveType = ::justbuild::just_serve::ServeArchiveTreeRequest_ArchiveType; switch (type) { case ServeArchiveType::ServeArchiveTreeRequest_ArchiveType_ZIP: { return "zip"; } case ServeArchiveType::ServeArchiveTreeRequest_ArchiveType_TAR: default: return "archive"; // default to .tar archive } } auto SymlinksResolveToPragmaSpecial( ::justbuild::just_serve::ServeArchiveTreeRequest_SymlinksResolve const& resolve) -> std::optional { using ServeSymlinksResolve = ::justbuild::just_serve::ServeArchiveTreeRequest_SymlinksResolve; switch (resolve) { case ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_IGNORE: { return PragmaSpecial::Ignore; } case ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_PARTIAL: { return PragmaSpecial::ResolvePartially; } case ServeSymlinksResolve:: ServeArchiveTreeRequest_SymlinksResolve_COMPLETE: { return PragmaSpecial::ResolveCompletely; } case ServeSymlinksResolve::ServeArchiveTreeRequest_SymlinksResolve_NONE: default: return std::nullopt; // default to NONE } } /// \brief Extracts the archive of given type into the destination directory /// provided. Returns nullopt on success, or error string on failure. [[nodiscard]] auto ExtractArchive(std::filesystem::path const& archive, std::string const& repo_type, std::filesystem::path const& dst_dir) noexcept -> std::optional { if (repo_type == "archive") { return ArchiveOps::ExtractArchive( ArchiveType::TarAuto, archive, dst_dir); } if (repo_type == "zip") { return ArchiveOps::ExtractArchive( ArchiveType::ZipAuto, archive, dst_dir); } return "unrecognized archive type"; } } // namespace auto SourceTreeService::EnsureGitCacheRoot() -> expected { // make sure the git root directory is properly initialized if (not FileSystemManager::CreateDirectory( native_context_->storage_config->GitRoot())) { return unexpected{ fmt::format("Could not create Git cache root {}", native_context_->storage_config->GitRoot().string())}; } if (not GitRepo::InitAndOpen(native_context_->storage_config->GitRoot(), true)) { return unexpected{ fmt::format("Could not initialize Git cache repository {}", native_context_->storage_config->GitRoot().string())}; } return std::monostate{}; } auto SourceTreeService::GetSubtreeFromCommit( std::filesystem::path const& repo_path, std::string const& commit, std::string const& subdir, std::shared_ptr const& logger) -> expected { if (auto git_cas = GitCAS::Open(repo_path)) { if (auto repo = GitRepo::Open(git_cas)) { // wrap logger for GitRepo call auto wrapped_logger = std::make_shared( [logger, repo_path, commit, subdir](auto const& msg, bool fatal) { if (fatal) { logger->Emit(LogLevel::Debug, "While retrieving subtree {} of commit {} " "from repository {}:\n{}", subdir, commit, repo_path.string(), msg); } }); return repo->GetSubtreeFromCommit(commit, subdir, wrapped_logger); } } return unexpected{GitLookupError::Fatal}; } auto SourceTreeService::GetSubtreeFromTree( std::filesystem::path const& repo_path, std::string const& tree_id, std::string const& subdir, std::shared_ptr const& logger) -> expected { if (auto git_cas = GitCAS::Open(repo_path)) { if (auto repo = GitRepo::Open(git_cas)) { // wrap logger for GitRepo call auto wrapped_logger = std::make_shared( [logger, repo_path, tree_id, subdir](auto const& msg, bool fatal) { if (fatal) { logger->Emit(LogLevel::Debug, "While retrieving subtree {} of tree {} " "from repository {}:\n{}", subdir, tree_id, repo_path.string(), msg); } }); if (auto subtree_id = repo->GetSubtreeFromTree(tree_id, subdir, wrapped_logger)) { return *subtree_id; } return unexpected{GitLookupError::NotFound}; // non-fatal failure } } return unexpected{GitLookupError::Fatal}; } auto SourceTreeService::GetBlobFromRepo(std::filesystem::path const& repo_path, std::string const& blob_id, std::shared_ptr const& logger) -> expected { if (auto git_cas = GitCAS::Open(repo_path)) { if (auto repo = GitRepo::Open(git_cas)) { // wrap logger for GitRepo call auto wrapped_logger = std::make_shared( [logger, repo_path, blob_id](auto const& msg, bool fatal) { if (fatal) { logger->Emit(LogLevel::Debug, "While checking existence of blob {} in " "repository {}:\n{}", blob_id, repo_path.string(), msg); } }); auto res = repo->TryReadBlob(blob_id, wrapped_logger); if (not res.first) { return unexpected{GitLookupError::Fatal}; } if (not res.second) { logger->Emit(LogLevel::Debug, "Blob {} not found in repository {}", blob_id, repo_path.string()); return unexpected{ GitLookupError::NotFound}; // non-fatal failure } return res.second.value(); } } // failed to open repository logger->Emit( LogLevel::Debug, "Failed to open repository {}", repo_path.string()); return unexpected{GitLookupError::Fatal}; } auto SourceTreeService::ServeCommitTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::ServeCommitTreeRequest* request, ServeCommitTreeResponse* response) -> ::grpc::Status { logger_->Emit( LogLevel::Debug, "ServeCommitTree({}, ...)", request->commit()); // get lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const& commit{request->commit()}; auto const& subdir{request->subdir()}; // try in local build root Git cache auto res = GetSubtreeFromCommit( native_context_->storage_config->GitRoot(), commit, subdir, logger_); if (res) { auto const tree_id = *std::move(res); auto status = ServeCommitTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas( tree_id, native_context_->storage_config->GitRoot()); if (status == ServeCommitTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } *(response->mutable_tree()) = tree_id; response->set_status(status); return ::grpc::Status::OK; } // report fatal failure if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed while retrieving subtree {} of commit {} from " "repository {}", subdir, commit, native_context_->storage_config->GitRoot().string()); response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // try given extra repositories, in order for (auto const& path : serve_config_.known_repositories) { auto res = GetSubtreeFromCommit(path, commit, subdir, logger_); if (res) { auto const tree_id = *std::move(res); auto status = ServeCommitTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas(tree_id, path); if (status == ServeCommitTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } *(response->mutable_tree()) = tree_id; response->set_status(status); return ::grpc::Status::OK; } // report fatal failure if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed while retrieving subtree {} of commit {} " "from repository {}", subdir, commit, path.string()); response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } // commit not found response->set_status(ServeCommitTreeResponse::NOT_FOUND); return ::grpc::Status::OK; } auto SourceTreeService::SyncArchive(std::string const& tree_id, std::filesystem::path const& repo_path, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status { auto status = ServeArchiveTreeResponse::OK; if (sync_tree) { status = SyncGitEntryToCas( tree_id, repo_path); if (status == ServeArchiveTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } *(response->mutable_tree()) = tree_id; response->set_status(status); return ::grpc::Status::OK; } template auto SourceTreeService::SyncGitEntryToCas( std::string const& object_hash, std::filesystem::path const& repo_path) const noexcept -> std::remove_cvref_t { // get gc locks for the local storages auto native_lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not native_lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); return TResponse::INTERNAL_ERROR; } std::optional compat_lock = std::nullopt; if (compat_context_ != nullptr) { compat_lock = GarbageCollector::SharedLock(*compat_context_->storage_config); if (not compat_lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); return TResponse::INTERNAL_ERROR; } } auto const hash_type = native_context_->storage_config->hash_function.GetType(); if (IsTreeObject(kType) and not ProtocolTraits::IsTreeAllowed(hash_type)) { logger_->Emit(LogLevel::Error, "Cannot sync tree {} from repository {} with " "the remote in compatible mode", object_hash, repo_path.string()); return TResponse::SYNC_ERROR; } auto repo = RepositoryConfig{}; if (not repo.SetGitCAS(repo_path, native_context_->storage_config)) { logger_->Emit( LogLevel::Error, "Failed to SetGitCAS at {}", repo_path.string()); return TResponse::INTERNAL_ERROR; } auto const digest = ArtifactDigestFactory::Create( hash_type, object_hash, 0, IsTreeObject(kType)); if (not digest) { logger_->Emit(LogLevel::Error, "SyncGitEntryToCas: {}", digest.error()); return TResponse::INTERNAL_ERROR; } auto const is_compat = compat_context_ != nullptr; auto git_api = MRGitApi{&repo, native_context_->storage_config, is_compat ? &*compat_context_->storage_config : nullptr, is_compat ? &*apis_.local : nullptr}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = *digest, .type = kType}}, *apis_.remote)) { logger_->Emit(LogLevel::Error, "Failed to sync object {} from repository {}", object_hash, repo_path.string()); return TResponse::SYNC_ERROR; } return TResponse::OK; } template auto SourceTreeService::SetDigestInResponse( gsl::not_null const& response, std::string const& object_hash, bool is_tree, bool from_git) const noexcept -> std::remove_cvref_t { // set digest in response auto native_digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), object_hash, /*size is unknown*/ 0, is_tree); if (not native_digest) { logger_->Emit(LogLevel::Error, "SetDigestInResponse: {}", std::move(native_digest).error()); return TResponse::INTERNAL_ERROR; } // in native mode, set the native digest in response if (ProtocolTraits::IsNative(apis_.remote->GetHashType())) { *(response->mutable_digest()) = ArtifactDigestFactory::ToBazel(*std::move(native_digest)); } else { // in compatible mode, we need to respond with a compatible digest if (compat_context_ == nullptr) { // sanity check logger_->Emit(LogLevel::Error, "Compatible storage not available as required"); return TResponse::INTERNAL_ERROR; } // get gc locks for the local storages auto native_lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not native_lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); return TResponse::INTERNAL_ERROR; } auto compat_lock = GarbageCollector::SharedLock(*compat_context_->storage_config); if (not compat_lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); return TResponse::INTERNAL_ERROR; } // get the compatible digest from the mapping that was created during // upload from Git cache auto const cached_obj = RehashUtils::ReadRehashedDigest(*native_digest, *native_context_->storage_config, *compat_context_->storage_config, from_git); if (not cached_obj) { logger_->Emit( LogLevel::Error, "SetDigestInResponse: {}", cached_obj.error()); return TResponse::INTERNAL_ERROR; } if (not *cached_obj) { logger_->Emit( LogLevel::Error, "Cached compatible object for native digest {} not found", native_digest->hash()); return TResponse::INTERNAL_ERROR; } // set compatible digest in response *(response->mutable_digest()) = ArtifactDigestFactory::ToBazel(cached_obj->value().digest); } return TResponse::OK; } auto SourceTreeService::ResolveContentTree( std::string const& tree_id, std::filesystem::path const& repo_path, bool repo_is_git_cache, std::optional const& resolve_special, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status { if (resolve_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( *native_context_->storage_config, tree_id, *resolve_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); if (not resolved_tree_id) { logger_->Emit(LogLevel::Error, "Failed to read resolved tree id from file {}", tree_id_file.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } return SyncArchive( *resolved_tree_id, repo_path, sync_tree, response); } // resolve tree; target repository is always the Git cache auto target_cas = GitCAS::Open(native_context_->storage_config->GitRoot()); if (not target_cas) { logger_->Emit(LogLevel::Error, "Failed to open Git ODB at {}", native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto source_cas = target_cas; if (not repo_is_git_cache) { source_cas = GitCAS::Open(repo_path); if (not source_cas) { logger_->Emit(LogLevel::Error, "Failed to open Git ODB at {}", repo_path.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } std::optional resolved_tree = std::nullopt; bool failed{false}; { TaskSystem ts{serve_config_.jobs}; resolve_symlinks_map_.ConsumeAfterKeysReady( &ts, {GitObjectToResolve{tree_id, ".", *resolve_special, /*known_info=*/std::nullopt, source_cas, target_cas}}, [&resolved_tree](auto hashes) { resolved_tree = *hashes[0]; }, [logger = logger_, tree_id, &failed](auto const& msg, bool fatal) { logger->Emit(LogLevel::Error, "While resolving tree {}:\n{}", tree_id, msg); failed = failed or fatal; }); } if (failed) { logger_->Emit( LogLevel::Error, "Failed to resolve tree id {}", tree_id); response->set_status(ServeArchiveTreeResponse::RESOLVE_ERROR); return ::grpc::Status::OK; } // check if we have a value if (not resolved_tree) { // check for cycles if (auto error = DetectAndReportCycle( fmt::format("resolving symlinks in tree {}", tree_id), resolve_symlinks_map_, kGitObjectToResolvePrinter)) { logger_->Emit(LogLevel::Error, *error); response->set_status(ServeArchiveTreeResponse::RESOLVE_ERROR); return ::grpc::Status::OK; } logger_->Emit(LogLevel::Error, "Unknown error while resolving tree id {}", tree_id); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // keep tree alive in the Git cache via a tagged commit auto wrapped_logger = std::make_shared( [logger = logger_, storage_config = native_context_->storage_config, resolved_tree](auto const& msg, bool fatal) { if (fatal) { logger->Emit(LogLevel::Error, "While keeping tree {} in repository {}:\n{}", resolved_tree->id, storage_config->GitRoot().string(), msg); } }); { // this is a non-thread-safe Git operation, so it must be guarded! std::unique_lock slock{*lock_}; // open real repository at Git CAS location auto git_repo = GitRepo::Open(native_context_->storage_config->GitRoot()); if (not git_repo) { logger_->Emit( LogLevel::Error, "Failed to open Git CAS repository {}", native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::RESOLVE_ERROR); return ::grpc::Status::OK; } // Important: message must be consistent with just-mr! if (not git_repo->KeepTree(resolved_tree->id, "Keep referenced tree alive", // message wrapped_logger)) { response->set_status(ServeArchiveTreeResponse::RESOLVE_ERROR); return ::grpc::Status::OK; } } // cache the resolved tree association if (not StorageUtils::WriteTreeIDFile(tree_id_file, resolved_tree->id)) { logger_->Emit(LogLevel::Error, "Failed to write resolved tree id to file {}", tree_id_file.string()); response->set_status(ServeArchiveTreeResponse::RESOLVE_ERROR); return ::grpc::Status::OK; } return SyncArchive(resolved_tree->id, repo_path, sync_tree, response); } // if no special handling of symlinks, use given tree as-is return SyncArchive(tree_id, repo_path, sync_tree, response); } auto SourceTreeService::ArchiveImportToGit( std::filesystem::path const& unpack_path, std::filesystem::path const& archive_tree_id_file, std::string const& content, std::string const& archive_type, std::string const& subdir, std::optional const& resolve_special, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status { // Important: commit message must match that in just-mr! auto res = GitRepo::ImportToGit( *native_context_->storage_config, unpack_path, /*commit_message=*/ fmt::format("Content of {} {}", archive_type, content), lock_); if (not res) { // report the error logger_->Emit(LogLevel::Error, "{}", res.error()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const& tree_id = *res; // write to tree id file if (not StorageUtils::WriteTreeIDFile(archive_tree_id_file, tree_id)) { logger_->Emit(LogLevel::Error, "Failed to write tree id to file {}", archive_tree_id_file.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // open the Git CAS repo auto just_git_cas = GitCAS::Open(native_context_->storage_config->GitRoot()); if (not just_git_cas) { logger_->Emit(LogLevel::Error, "Failed to open Git ODB at {}", native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto just_git_repo = GitRepo::Open(just_git_cas); if (not just_git_repo) { logger_->Emit(LogLevel::Error, "Failed to open Git repository {}", native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // wrap logger for GitRepo call std::string err; auto wrapped_logger = std::make_shared( [&err, subdir, tree_id](auto const& msg, bool fatal) { if (fatal) { err = fmt::format("While retrieving subtree {} of tree {}:\n{}", subdir, tree_id, msg); } }); // get the subtree id; this is thread-safe auto subtree_id = just_git_repo->GetSubtreeFromTree(tree_id, subdir, wrapped_logger); if (not subtree_id) { logger_->Emit(LogLevel::Error, "{}", err); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } return ResolveContentTree(*subtree_id, native_context_->storage_config->GitRoot(), /*repo_is_git_cache=*/true, resolve_special, sync_tree, response); } auto SourceTreeService::ServeArchiveTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::ServeArchiveTreeRequest* request, ServeArchiveTreeResponse* response) -> ::grpc::Status { logger_->Emit( LogLevel::Debug, "ServeArchiveTree({}, ...)", request->content()); // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const& content{request->content()}; auto archive_type = ArchiveTypeToString(request->archive_type()); auto const& subdir{request->subdir()}; auto resolve_special = SymlinksResolveToPragmaSpecial(request->resolve_symlinks()); // check for archive_tree_id_file auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( *native_context_->storage_config, archive_type, content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file auto archive_tree_id = FileSystemManager::ReadFile(archive_tree_id_file); if (not archive_tree_id) { logger_->Emit(LogLevel::Error, "Failed to read tree id from file {}", archive_tree_id_file.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check local build root Git cache auto res = GetSubtreeFromTree(native_context_->storage_config->GitRoot(), *archive_tree_id, subdir, logger_); if (res) { return ResolveContentTree( *res, // tree_id native_context_->storage_config->GitRoot(), /*repo_is_git_cache=*/true, resolve_special, request->sync_tree(), response); } // check for fatal error if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed to open repository {}", native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check known repositories for (auto const& path : serve_config_.known_repositories) { auto res = GetSubtreeFromTree(path, *archive_tree_id, subdir, logger_); if (res) { return ResolveContentTree(*res, // tree_id path, /*repo_is_git_cache=*/false, resolve_special, request->sync_tree(), response); } // check for fatal error if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed to open repository {}", path.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } // report error for missing tree specified in id file logger_->Emit(LogLevel::Error, "Failed while retrieving subtree {} of known tree {}", subdir, *archive_tree_id); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // acquire lock for native CAS auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check if content is in native local CAS already auto const digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), content, 0, /*is_tree=*/false); auto const& native_cas = native_context_->storage->CAS(); auto content_cas_path = digest ? native_cas.BlobPath(*digest, /*is_executable=*/false) : std::nullopt; if (not content_cas_path) { // check if content blob is in Git cache auto res = GetBlobFromRepo( native_context_->storage_config->GitRoot(), content, logger_); if (res) { // add to native CAS content_cas_path = StorageUtils::AddToCAS(*native_context_->storage, *res); } if (res.error() == GitLookupError::Fatal) { logger_->Emit( LogLevel::Error, "Failed while trying to retrieve content {} from repository {}", content, native_context_->storage_config->GitRoot().string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } if (not content_cas_path) { // check if content blob is in a known repository for (auto const& path : serve_config_.known_repositories) { auto res = GetBlobFromRepo(path, content, logger_); if (res) { // add to native CAS content_cas_path = StorageUtils::AddToCAS(*native_context_->storage, *res); if (content_cas_path) { break; } } if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed while trying to retrieve content {} from " "repository {}", content, path.string()); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } } if (digest and not content_cas_path) { // try to retrieve it from remote CAS if (not(apis_.remote->IsAvailable(*digest) and apis_.remote->RetrieveToCas( {Artifact::ObjectInfo{.digest = *digest, .type = ObjectType::File}}, *apis_.local))) { // content could not be found response->set_status(ServeArchiveTreeResponse::NOT_FOUND); return ::grpc::Status::OK; } // content should now be in native CAS content_cas_path = native_cas.BlobPath(*digest, /*is_executable=*/false); if (not content_cas_path) { logger_->Emit( LogLevel::Error, "Retrieving content {} from native CAS failed unexpectedly", content); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } // extract archive auto tmp_dir = native_context_->storage_config->CreateTypedTmpDir(archive_type); if (not tmp_dir) { logger_->Emit( LogLevel::Error, "Failed to create tmp path for {} archive with content {}", archive_type, content); response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto res = ExtractArchive(*content_cas_path, archive_type, tmp_dir->GetPath()); if (res != std::nullopt) { logger_->Emit(LogLevel::Error, "Failed to extract archive {} from native CAS:\n{}", content_cas_path->string(), *res); response->set_status(ServeArchiveTreeResponse::UNPACK_ERROR); return ::grpc::Status::OK; } // import to git return ArchiveImportToGit(tmp_dir->GetPath(), archive_tree_id_file, content, archive_type, subdir, resolve_special, request->sync_tree(), response); } auto SourceTreeService::DistdirImportToGit( std::string const& distdir_tree_id, std::string const& content_id, std::unordered_map> const& content_list, bool sync_tree, ServeDistdirTreeResponse* response) -> ::grpc::Status { // create tmp directory for the distdir auto distdir_tmp_dir = native_context_->storage_config->CreateTypedTmpDir("distdir"); if (not distdir_tmp_dir) { logger_->Emit(LogLevel::Error, "Failed to create tmp path for distdir target {}", content_id); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const& tmp_path = distdir_tmp_dir->GetPath(); // link the native CAS blobs into the tmp dir auto const& native_cas = native_context_->storage->CAS(); if (not std::all_of( content_list.begin(), content_list.end(), [&native_cas, tmp_path](auto const& kv) { auto const digest = ArtifactDigestFactory::Create( native_cas.GetHashFunction().GetType(), kv.second.first, 0, /*is_tree=*/false); if (not digest) { return false; } auto content_path = native_cas.BlobPath(*digest, kv.second.second); if (content_path) { return FileSystemManager::CreateFileHardlink( *content_path, // from: cas_path/content_id tmp_path / kv.first) // to: tmp_path/name .has_value(); } return false; })) { logger_->Emit(LogLevel::Error, "Failed to create links to native CAS content {}", content_id); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // Important: commit message must match that in just-mr! auto res = GitRepo::ImportToGit( *native_context_->storage_config, tmp_path, /*commit_message=*/fmt::format("Content of distdir {}", content_id), lock_); if (not res) { // report the error logger_->Emit(LogLevel::Error, "{}", res.error()); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto tree_id = *std::move(res); // check the committed tree matches what we expect if (tree_id != distdir_tree_id) { // something is very wrong... logger_->Emit(LogLevel::Error, "Unexpected mismatch for tree of committed " "distdir:\nexpected {} but got {}", distdir_tree_id, tree_id); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // if asked, sync tree (and implicitly all blobs) with remote CAS auto status = ServeDistdirTreeResponse::OK; if (sync_tree) { status = SyncGitEntryToCas( tree_id, native_context_->storage_config->GitRoot()); if (status == ServeDistdirTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } // set response on success *(response->mutable_tree()) = std::move(tree_id); response->set_status(status); return ::grpc::Status::OK; } auto SourceTreeService::ServeDistdirTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::ServeDistdirTreeRequest* request, ServeDistdirTreeResponse* response) -> ::grpc::Status { logger_->Emit(LogLevel::Debug, "ServeDistdirTree(...)"); // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // acquire lock for native CAS auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // create in-memory tree from distfiles map GitRepo::tree_entries_t entries{}; entries.reserve(request->distfiles().size()); auto const& native_cas = native_context_->storage->CAS(); std::unordered_map> content_list{}; content_list.reserve(request->distfiles().size()); bool const is_native = ProtocolTraits::IsNative(apis_.remote->GetHashType()); for (auto const& kv : request->distfiles()) { bool blob_found{}; std::string blob_digest; // The digest of the requested distfile, taken // by the hash applicable for our CAS; this // might be different from content, if our CAS // is not based on git blob identifiers // (i.e., if we're not in native mode). auto const& content = kv.content(); // check content blob is known // first check the native local CAS itself, provided it uses the same // type of identifier auto const digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), content, 0, /*is_tree=*/false); if (is_native) { blob_found = digest and native_cas.BlobPath(*digest, kv.executable()); } if (blob_found) { blob_digest = content; } else { // check local Git cache auto res = GetBlobFromRepo( native_context_->storage_config->GitRoot(), content, logger_); if (res) { // add content to native local CAS auto stored_blob = native_cas.StoreBlob(*res, kv.executable()); if (not stored_blob) { logger_->Emit(LogLevel::Error, "Failed to store content {} from local Git " "cache to native local CAS", content); response->set_status( ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } blob_found = true; blob_digest = stored_blob->hash(); } else { if (res.error() == GitLookupError::Fatal) { logger_->Emit( LogLevel::Error, "Failed while trying to retrieve content {} from " "repository {}", content, native_context_->storage_config->GitRoot().string()); response->set_status( ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check known repositories for (auto const& path : serve_config_.known_repositories) { auto res = GetBlobFromRepo(path, content, logger_); if (res) { // add content to native local CAS auto stored_blob = native_cas.StoreBlob(*res, kv.executable()); if (not stored_blob) { logger_->Emit( LogLevel::Error, "Failed to store content {} from known " "repository {} to native local CAS", path.string(), content); response->set_status( ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } blob_found = true; blob_digest = stored_blob->hash(); break; } if (res.error() == GitLookupError::Fatal) { logger_->Emit( LogLevel::Error, "Failed while trying to retrieve content {} from " "repository {}", content, path.string()); response->set_status( ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } if (not blob_found) { // check remote CAS if (is_native and digest and apis_.remote->IsAvailable(*digest)) { // retrieve content to native local CAS if (not apis_.remote->RetrieveToCas( {Artifact::ObjectInfo{ .digest = *digest, .type = kv.executable() ? ObjectType::Executable : ObjectType::File}}, *apis_.local)) { logger_->Emit(LogLevel::Error, "Failed to retrieve content {} from " "remote to native local CAS", content); response->set_status( ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } blob_found = true; blob_digest = content; } } } } // error out if blob is not known if (not blob_found) { logger_->Emit(LogLevel::Error, "Content {} is not known", content); response->set_status(ServeDistdirTreeResponse::NOT_FOUND); return ::grpc::Status::OK; } // store content blob to the entries list, using the expected raw id if (auto raw_id = FromHexString(content)) { entries[*raw_id].emplace_back( kv.name(), kv.executable() ? ObjectType::Executable : ObjectType::File); } else { logger_->Emit( LogLevel::Error, "Conversion of content {} to raw id failed unexpectedly", content); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // store to content_list for import-to-git hardlinking content_list.insert_or_assign( kv.name(), std::make_pair(blob_digest, kv.executable())); } // get hash of distdir content; this must match with that in just-mr auto content_id = HashFunction{HashFunction::Type::GitSHA1} .HashBlobData(nlohmann::json(content_list).dump()) .HexString(); // create in-memory tree of the distdir, now that we know we have all blobs auto tree = GitRepo::CreateShallowTree(entries); if (not tree) { logger_->Emit(LogLevel::Error, "Failed to construct in-memory tree for distdir"); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get hash from raw_id auto tree_id = ToHexString(tree->first); // add tree to native local CAS if (not native_cas.StoreTree(tree->second)) { logger_->Emit(LogLevel::Error, "Failed to store distdir tree {} to native local CAS", tree_id); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check if tree is already in Git cache auto has_tree = GitRepo::IsTreeInRepo( native_context_->storage_config->GitRoot(), tree_id); if (not has_tree) { logger_->Emit(LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, native_context_->storage_config->GitRoot().string(), std::move(has_tree).error()); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // if asked, sync tree and all blobs with remote CAS auto status = ServeDistdirTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas( tree_id, native_context_->storage_config->GitRoot()); if (status == ServeDistdirTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } // set response on success *(response->mutable_tree()) = std::move(tree_id); response->set_status(status); return ::grpc::Status::OK; } // check if tree is in a known repository for (auto const& path : serve_config_.known_repositories) { auto has_tree = GitRepo::IsTreeInRepo(path, tree_id); if (not has_tree) { logger_->Emit( LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, path.string(), *std::move(has_tree)); response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // if asked, sync tree and all blobs with remote CAS auto status = ServeDistdirTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas(tree_id, path); if (status == ServeDistdirTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } } // set response on success *(response->mutable_tree()) = std::move(tree_id); response->set_status(status); return ::grpc::Status::OK; } } // otherwise, we import the tree from native local CAS ourselves return DistdirImportToGit( tree_id, content_id, content_list, request->sync_tree(), response); } auto SourceTreeService::ServeContent( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::ServeContentRequest* request, ServeContentResponse* response) -> ::grpc::Status { auto const& content{request->content()}; logger_->Emit(LogLevel::Debug, "ServeContent({})", content); // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get gc lock for native storage auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check if content blob is in Git cache auto res = GetBlobFromRepo( native_context_->storage_config->GitRoot(), content, logger_); if (res) { auto status = SyncGitEntryToCas( content, native_context_->storage_config->GitRoot()); if (status == ServeContentResponse::OK) { status = SetDigestInResponse( response, content, /*is_tree=*/false, /*from_git=*/true); } response->set_status(status); return ::grpc::Status::OK; } if (res.error() == GitLookupError::Fatal) { logger_->Emit(LogLevel::Error, "Failed while checking for content {} in repository {}", content, native_context_->storage_config->GitRoot().string()); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check if content blob is in a known repository for (auto const& path : serve_config_.known_repositories) { auto res = GetBlobFromRepo(path, content, logger_); if (res) { // upload blob to remote CAS auto status = SyncGitEntryToCas( content, path); if (status == ServeContentResponse::OK) { status = SetDigestInResponse( response, content, /*is_tree=*/false, /*from_git=*/true); } response->set_status(status); return ::grpc::Status::OK; } if (res.error() == GitLookupError::Fatal) { auto str = fmt::format( "Failed while checking for content {} in repository {}", content, path.string()); logger_->Emit(LogLevel::Error, str); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } } // check also in the native local CAS auto const native_digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), content, /*size is unknown*/ 0, /*is_tree=*/false); if (not native_digest) { logger_->Emit(LogLevel::Error, "Failed to create digest object"); response->set_status(ServeContentResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (apis_.local->IsAvailable(*native_digest)) { // upload blob to remote CAS if (not apis_.local->RetrieveToCas( {Artifact::ObjectInfo{.digest = *native_digest, .type = ObjectType::File}}, *apis_.remote)) { logger_->Emit(LogLevel::Error, "Failed to sync content {} from local native CAS", content); response->set_status(ServeContentResponse::SYNC_ERROR); return ::grpc::Status::OK; } auto const status = SetDigestInResponse( response, content, /*is_tree=*/false, /*from_git=*/false); response->set_status(status); return ::grpc::Status::OK; } // content blob not known response->set_status(ServeContentResponse::NOT_FOUND); return ::grpc::Status::OK; } auto SourceTreeService::ServeTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::ServeTreeRequest* request, ServeTreeResponse* response) -> ::grpc::Status { auto const& tree_id{request->tree()}; // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get gc lock for native storage auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check if tree is in Git cache auto has_tree = GitRepo::IsTreeInRepo( native_context_->storage_config->GitRoot(), tree_id); if (not has_tree) { logger_->Emit(LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, native_context_->storage_config->GitRoot().string(), std::move(has_tree).error()); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // upload tree to remote CAS auto status = SyncGitEntryToCas( tree_id, native_context_->storage_config->GitRoot()); if (status == ServeTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } response->set_status(status); return ::grpc::Status::OK; } // check if tree is in a known repository for (auto const& path : serve_config_.known_repositories) { auto has_tree = GitRepo::IsTreeInRepo(path, tree_id); if (not has_tree) { logger_->Emit( LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, path.string(), std::move(has_tree).error()); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // upload blob to remote CAS auto status = SyncGitEntryToCas(tree_id, path); if (status == ServeTreeResponse::OK) { status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/true); } response->set_status(status); return ::grpc::Status::OK; } } // check also in the native local CAS auto const native_digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), tree_id, /*size is unknown*/ 0, /*is_tree=*/true); if (not native_digest) { logger_->Emit(LogLevel::Error, "Failed to create digest object"); response->set_status(ServeTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (apis_.local->IsAvailable(*native_digest)) { // upload tree to remote CAS if (not apis_.local->RetrieveToCas( {Artifact::ObjectInfo{.digest = *native_digest, .type = ObjectType::Tree}}, *apis_.remote)) { logger_->Emit(LogLevel::Error, "Failed to sync tree {} from native local CAS", tree_id); response->set_status(ServeTreeResponse::SYNC_ERROR); return ::grpc::Status::OK; } auto const status = SetDigestInResponse( response, tree_id, /*is_tree=*/true, /*from_git=*/false); response->set_status(status); return ::grpc::Status::OK; } // tree not known response->set_status(ServeTreeResponse::NOT_FOUND); return ::grpc::Status::OK; } auto SourceTreeService::CheckRootTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::CheckRootTreeRequest* request, CheckRootTreeResponse* response) -> ::grpc::Status { auto const& tree_id{request->tree()}; logger_->Emit(LogLevel::Debug, "CheckRootTree({})", tree_id); // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get gc lock for native storage auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // check first in the Git cache auto has_tree = GitRepo::IsTreeInRepo( native_context_->storage_config->GitRoot(), tree_id); if (not has_tree) { logger_->Emit(LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, native_context_->storage_config->GitRoot().string(), std::move(has_tree).error()); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // success! response->set_status(CheckRootTreeResponse::OK); return ::grpc::Status::OK; } // check if tree is in a known repository for (auto const& path : serve_config_.known_repositories) { auto has_tree = GitRepo::IsTreeInRepo(path, tree_id); if (not has_tree) { logger_->Emit( LogLevel::Error, "Failed while checking for tree {} in repository {}:\n{}", tree_id, path.string(), std::move(has_tree).error()); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (*has_tree) { // success! response->set_status(CheckRootTreeResponse::OK); return ::grpc::Status::OK; } } // now check in the native local CAS auto const native_digest = ArtifactDigestFactory::Create( native_context_->storage_config->hash_function.GetType(), tree_id, 0, /*is_tree=*/true); if (not native_digest) { logger_->Emit(LogLevel::Error, "Failed to create digest object"); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (native_context_->storage->CAS().TreePath(*native_digest)) { // As we currently build only against roots in Git repositories, we need // to move the tree from CAS to local Git storage auto tmp_dir = native_context_->storage_config->CreateTypedTmpDir( "source-tree-check-root-tree"); if (not tmp_dir) { logger_->Emit(LogLevel::Error, "Failed to create tmp directory for copying git-tree " "{} from remote CAS", tree_id); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (not apis_.local->RetrieveToPaths( {Artifact::ObjectInfo{.digest = *native_digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { logger_->Emit(LogLevel::Error, "Failed to copy git-tree {} to {}", tree_id, tmp_dir->GetPath().string()); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // Import from tmp dir to Git cache auto res = GitRepo::ImportToGit( *native_context_->storage_config, tmp_dir->GetPath(), /*commit_message=*/fmt::format("Content of tree {}", tree_id), lock_); if (not res) { // report the error logger_->Emit(LogLevel::Error, "{}", res.error()); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const& imported_tree_id = *res; // sanity check if (imported_tree_id != tree_id) { logger_->Emit( LogLevel::Error, "Unexpected mismatch in imported tree:\nexpected {} but got {}", tree_id, imported_tree_id); response->set_status(CheckRootTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // success! response->set_status(CheckRootTreeResponse::OK); return ::grpc::Status::OK; } // tree not known response->set_status(CheckRootTreeResponse::NOT_FOUND); return ::grpc::Status::OK; } auto SourceTreeService::GetRemoteTree( ::grpc::ServerContext* /* context */, const ::justbuild::just_serve::GetRemoteTreeRequest* request, GetRemoteTreeResponse* response) -> ::grpc::Status { logger_->Emit( LogLevel::Debug, "GetRemoteTree({})", request->digest().hash()); // get gc lock for Git cache auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(GetRemoteTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(GetRemoteTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get gc lock for native storage auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(GetRemoteTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get tree from remote CAS into tmp dir auto const remote_digest = ArtifactDigestFactory::FromBazel( apis_.remote->GetHashType(), request->digest()); if (not remote_digest or not apis_.remote->IsAvailable(*remote_digest)) { logger_->Emit(LogLevel::Error, "Remote CAS does not contain expected tree {}", request->digest().hash()); response->set_status(GetRemoteTreeResponse::FAILED_PRECONDITION); return ::grpc::Status::OK; } auto tmp_dir = native_context_->storage_config->CreateTypedTmpDir( "source-tree-get-remote-tree"); if (not tmp_dir) { logger_->Emit(LogLevel::Error, "Failed to create tmp directory for copying git-tree {} " "from remote CAS", remote_digest->hash()); response->set_status(GetRemoteTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (not apis_.remote->ParallelRetrieveToCas( {Artifact::ObjectInfo{.digest = *remote_digest, .type = ObjectType::Tree}}, *apis_.local, serve_config_.jobs, true)) { logger_->Emit( LogLevel::Error, "Failed to parallel retrieve tree {} from remote CAS to local CAS", remote_digest->hash()); response->set_status(GetRemoteTreeResponse::FAILED_PRECONDITION); return ::grpc::Status::OK; } if (not apis_.local->RetrieveToPaths( {Artifact::ObjectInfo{.digest = *remote_digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { logger_->Emit(LogLevel::Error, "Failed to install tree {} from local CAS", remote_digest->hash()); response->set_status(GetRemoteTreeResponse::FAILED_PRECONDITION); return ::grpc::Status::OK; } // Import from tmp dir to Git cache auto res = GitRepo::ImportToGit( *native_context_->storage_config, tmp_dir->GetPath(), /*commit_message=*/ fmt::format("Content of tree {}", remote_digest->hash()), lock_); if (not res) { // report the error logger_->Emit(LogLevel::Error, "{}", res.error()); response->set_status(GetRemoteTreeResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } logger_->Emit(LogLevel::Debug, "GetRemoteTree: imported tree {} to Git as {}", remote_digest->hash(), *res); // success! response->set_status(GetRemoteTreeResponse::OK); return ::grpc::Status::OK; } auto SourceTreeService::ComputeTreeStructure( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ComputeTreeStructureRequest* request, ComputeTreeStructureResponse* response) -> ::grpc::Status { logger_->Emit(LogLevel::Debug, "GetTreeStructure({})", request->tree()); auto repo_lock = RepositoryGarbageCollector::SharedLock( *native_context_->storage_config); if (not repo_lock) { logger_->Emit(LogLevel::Error, "Could not acquire repo gc SharedLock"); response->set_status(ComputeTreeStructureResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // ensure Git cache exists if (auto done = EnsureGitCacheRoot(); not done) { logger_->Emit(LogLevel::Error, std::move(done).error()); response->set_status(ComputeTreeStructureResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } // get gc lock for native storage auto lock = GarbageCollector::SharedLock(*native_context_->storage_config); if (not lock) { logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); response->set_status(ComputeTreeStructureResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto const tree_digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, request->tree(), /*size_unknown=*/0, /*is_tree=*/true); if (not tree_digest) { logger_->Emit(LogLevel::Error, tree_digest.error()); response->set_status(ComputeTreeStructureResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } auto known_repositories = serve_config_.known_repositories; known_repositories.push_back(native_context_->storage_config->GitRoot()); std::optional tree_structure; if (auto from_local = TreeStructureUtils::ComputeStructureLocally( *tree_digest, known_repositories, *native_context_->storage_config, lock_)) { tree_structure = std::move(from_local).value(); } else { // A critical error occurred: logger_->Emit(LogLevel::Error, std::move(from_local).error()); response->set_status(ComputeTreeStructureResponse::INTERNAL_ERROR); return ::grpc::Status::OK; } if (not tree_structure.has_value()) { logger_->Emit( LogLevel::Error, "Failed to find {}", tree_digest->hash()); response->set_status(ComputeTreeStructureResponse::NOT_FOUND); return ::grpc::Status::OK; } response->set_tree_structure_hash(tree_structure->hash()); response->set_status(ComputeTreeStructureResponse::OK); return ::grpc::Status::OK; } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/source_tree.hpp000066400000000000000000000243341516554100600322620ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_SERVE_SERVICE_SOURCE_TREE_HPP #define INCLUDED_SRC_BUILDTOOL_SERVE_API_SERVE_SERVICE_SOURCE_TREE_HPP #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "justbuild/just_serve/just_serve.pb.h" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/utils/cpp/expected.hpp" // Service for improved interaction with the target-level cache. class SourceTreeService final : public justbuild::just_serve::SourceTree::Service { public: using ServeCommitTreeResponse = ::justbuild::just_serve::ServeCommitTreeResponse; using ServeArchiveTreeResponse = ::justbuild::just_serve::ServeArchiveTreeResponse; using ServeDistdirTreeResponse = ::justbuild::just_serve::ServeDistdirTreeResponse; using ServeContentResponse = ::justbuild::just_serve::ServeContentResponse; using ServeTreeResponse = ::justbuild::just_serve::ServeTreeResponse; using CheckRootTreeResponse = ::justbuild::just_serve::CheckRootTreeResponse; using GetRemoteTreeResponse = ::justbuild::just_serve::GetRemoteTreeResponse; using ComputeTreeStructureResponse = ::justbuild::just_serve::ComputeTreeStructureResponse; explicit SourceTreeService( gsl::not_null const& serve_config, gsl::not_null const& apis, gsl::not_null const& native_context, gsl::not_null const& lock, LocalContext const* compat_context = nullptr) noexcept : serve_config_{*serve_config}, apis_{*apis}, lock_{lock}, native_context_{native_context}, compat_context_{compat_context} {} // Retrieve the Git-subtree identifier from a given Git commit. // // There are no method-specific errors. auto ServeCommitTree( ::grpc::ServerContext* context, const ::justbuild::just_serve::ServeCommitTreeRequest* request, ServeCommitTreeResponse* response) -> ::grpc::Status override; // Retrieve the Git-subtree identifier for the tree obtained // by unpacking an archive with a given blob identifier. // // There are no method-specific errors. auto ServeArchiveTree( ::grpc::ServerContext* context, const ::justbuild::just_serve::ServeArchiveTreeRequest* request, ServeArchiveTreeResponse* response) -> ::grpc::Status override; // Compute the Git-tree identifier for the tree containing the content // blobs of a list of distfiles. The implementation must only return the // tree identifier if ALL content blobs are known. // // There are no method-specific errors. auto ServeDistdirTree( ::grpc::ServerContext* context, const ::justbuild::just_serve::ServeDistdirTreeRequest* request, ServeDistdirTreeResponse* response) -> ::grpc::Status override; // Make a given content blob available in remote CAS, if blob is known. // // There are no method-specific errors. auto ServeContent( ::grpc::ServerContext* context, const ::justbuild::just_serve::ServeContentRequest* request, ServeContentResponse* response) -> ::grpc::Status override; // Make a given tree available in remote CAS, if tree is known. // // There are no method-specific errors. auto ServeTree(::grpc::ServerContext* context, const ::justbuild::just_serve::ServeTreeRequest* request, ServeTreeResponse* response) -> ::grpc::Status override; // Check if a Git-tree is locally known and, if found, make it available // in a location where this serve instance can build against. // The implementation should not interrogate the associated remote-execution // endpoint at any point during the completion of this request. // // There are no method-specific errors. auto CheckRootTree( ::grpc::ServerContext* context, const ::justbuild::just_serve::CheckRootTreeRequest* request, CheckRootTreeResponse* response) -> ::grpc::Status override; // Retrieve a given tree from the CAS of the associated remote-execution // endpoint and make it available in a location where this serve instance // can build against. // // There are no method-specific errors. auto GetRemoteTree( ::grpc::ServerContext* context, const ::justbuild::just_serve::GetRemoteTreeRequest* request, GetRemoteTreeResponse* response) -> ::grpc::Status override; // Compute the tree structure of the given tree and return the Git tree // identifier of the resulting structure. // // There are no method-specific errors. auto ComputeTreeStructure( ::grpc::ServerContext* context, const ::justbuild::just_serve::ComputeTreeStructureRequest* request, ComputeTreeStructureResponse* response) -> ::grpc::Status override; private: RemoteServeConfig const& serve_config_; ApiBundle const& apis_; gsl::not_null lock_; gsl::not_null native_context_; LocalContext const* compat_context_; std::shared_ptr logger_{std::make_shared("serve-service")}; // symlinks resolver map ResolveSymlinksMap resolve_symlinks_map_{CreateResolveSymlinksMap()}; /// \brief Ensure that the Git cache repository exists. /// \returns Error message on failure. [[nodiscard]] auto EnsureGitCacheRoot() -> expected; /// \brief Check if commit exists and tries to get the subtree if found. /// \returns The subtree hash on success or an unexpected error (fatal or /// commit was not found). [[nodiscard]] static auto GetSubtreeFromCommit( std::filesystem::path const& repo_path, std::string const& commit, std::string const& subdir, std::shared_ptr const& logger) -> expected; /// \brief Check if tree exists and tries to get the subtree if found. /// \returns The subtree hash on success or an unexpected error (fatal or /// subtree not found). [[nodiscard]] static auto GetSubtreeFromTree( std::filesystem::path const& repo_path, std::string const& tree_id, std::string const& subdir, std::shared_ptr const& logger) -> expected; /// \brief Tries to retrieve the blob from a repository. /// \returns The subtree hash on success or an unexpected error (fatal or /// blob not found). [[nodiscard]] static auto GetBlobFromRepo( std::filesystem::path const& repo_path, std::string const& blob_id, std::shared_ptr const& logger) -> expected; [[nodiscard]] auto SyncArchive(std::string const& tree_id, std::filesystem::path const& repo_path, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status; template [[nodiscard]] auto SyncGitEntryToCas(std::string const& object_hash, std::filesystem::path const& repo_path) const noexcept -> std::remove_cvref_t; /// \brief Set the digest field of a serve response. /// In compatible mode, this handles also the interaction with the storages /// to recover the corresponding compatible digest from a native digest, as /// stored in file mappings. template [[nodiscard]] auto SetDigestInResponse( gsl::not_null const& response, std::string const& object_hash, bool is_tree, bool from_git) const noexcept -> std::remove_cvref_t; /// \brief Resolves a tree from given repository with respect to symlinks. /// The resolved tree will always be placed in the Git cache. [[nodiscard]] auto ResolveContentTree( std::string const& tree_id, std::filesystem::path const& repo_path, bool repo_is_git_cache, std::optional const& resolve_special, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status; [[nodiscard]] auto ArchiveImportToGit( std::filesystem::path const& unpack_path, std::filesystem::path const& archive_tree_id_file, std::string const& content, std::string const& archive_type, std::string const& subdir, std::optional const& resolve_special, bool sync_tree, ServeArchiveTreeResponse* response) -> ::grpc::Status; [[nodiscard]] auto DistdirImportToGit( std::string const& tree_id, std::string const& content_id, std::unordered_map> const& content_list, bool sync_tree, ServeDistdirTreeResponse* response) -> ::grpc::Status; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_SERVE_SERVICE_SOURCE_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/target.cpp000066400000000000000000001351411516554100600312230ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/serve_api/serve_service/target.hpp" #include #include #include #include #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_file.hpp" #include "src/buildtool/main/analyse.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/main/build_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" #include "src/buildtool/serve_api/serve_service/target_utils.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/repository_garbage_collector.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { // Store the artifact root given by the target-cache entry. Return std::nullopt // on success and an error message on failure. auto KeepRoot(TargetCacheEntry const& target_entry, LocalContext const& local_context, gsl::not_null const& api, gsl::not_null const& tagging_lock) -> std::optional { auto cache_result = target_entry.ToResult(); if (not cache_result) { return "Failed to get analysis result for target cache entry."; } auto tmp_dir = local_context.storage_config->CreateTypedTmpDir("keep-artifacts-stage"); if (not tmp_dir) { return "Failed to create a temporary directory for keeping stage."; } std::vector paths{}; std::vector object_infos{}; for (auto const& [rel_path, artifact] : cache_result->artifact_stage->Map()) { paths.push_back(tmp_dir->GetPath() / rel_path); object_infos.push_back(*artifact->Artifact().ToArtifact().Info()); } if (not api->RetrieveToPaths(object_infos, paths)) { return fmt::format("Failed installing {} to temp dir {}.", cache_result->artifact_stage->ToString(), tmp_dir->GetPath().string()); } auto import_result = GitRepo::ImportToGit(*local_context.storage_config, tmp_dir->GetPath(), "Keep artifact stage", tagging_lock); if (not import_result) { return import_result.error(); } return std::nullopt; // No errors } } // namespace auto TargetService::GetDispatchList( ArtifactDigest const& dispatch_digest) noexcept -> expected, ::grpc::Status> { // get blob from remote cas auto const& dispatch_info = Artifact::ObjectInfo{.digest = dispatch_digest, .type = ObjectType::File}; if (not apis_.local->IsAvailable(dispatch_digest) and not apis_.remote->RetrieveToCas({dispatch_info}, *apis_.local)) { return unexpected{ ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, fmt::format("Could not retrieve blob {} from " "remote-execution endpoint", dispatch_info.ToString())}}; } // get blob content auto const& dispatch_str = apis_.local->RetrieveToMemory(dispatch_info); if (not dispatch_str) { // this should not fail unless something really broke... return unexpected{::grpc::Status{ ::grpc::StatusCode::INTERNAL, fmt::format("Unexpected failure in reading blob {} from CAS", dispatch_info.ToString())}}; } // parse content auto parsed = ParseDispatch(*dispatch_str); if (not parsed) { // pass the parsing error forward return unexpected{ ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, std::move(parsed).error()}}; } return *std::move(parsed); } auto TargetService::HandleFailureLog( std::filesystem::path const& logfile, std::string const& failure_scope, ::justbuild::just_serve::ServeTargetResponse* response) noexcept -> ::grpc::Status { logger_->Emit(LogLevel::Trace, [&logfile] { if (auto s = FileSystemManager::ReadFile(logfile)) { return *s; } return fmt::format("Failed to read failure log file {}", logfile.string()); }); // ...but try to give the client the proper log auto const& cas = local_context_.storage->CAS(); auto digest = cas.StoreBlob(logfile, /*is_executable=*/false); if (not digest) { auto msg = fmt::format("Failed to store log of failed {} to local CAS", failure_scope); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } // upload log blob to remote if (not apis_.local->RetrieveToCas( {Artifact::ObjectInfo{.digest = *digest, .type = ObjectType::File}}, *apis_.remote)) { auto msg = fmt::format("Failed to upload to remote CAS the failed {} log {}", failure_scope, digest->hash()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::UNAVAILABLE, msg}; } // set response with log digest (*response->mutable_log()) = ArtifactDigestFactory::ToBazel(*digest); return ::grpc::Status::OK; } auto TargetService::CreateRemoteExecutionConfig( const ::justbuild::just_serve::ServeTargetRequest* request) noexcept -> expected { // read in the execution properties auto platform_properties = ExecutionProperties{}; for (auto const& p : request->execution_properties()) { platform_properties[p.name()] = p.value(); } // read in the dispatch list auto const dispatch_info_digest = ArtifactDigestFactory::FromBazel( local_context_.storage_config->hash_function.GetType(), request->dispatch_info()); if (not dispatch_info_digest) { logger_->Emit(LogLevel::Error, "{}", dispatch_info_digest.error()); return unexpected{::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, dispatch_info_digest.error()}}; } auto res = GetDispatchList(*dispatch_info_digest); if (not res) { auto err = std::move(res).error(); logger_->Emit(LogLevel::Info, "{}", err.error_message()); return unexpected{std::move(err)}; } // the remote and cache addresses are kept from the stored ApiBundle return RemoteExecutionConfig{ .remote_address = remote_context_.exec_config->remote_address, .remote_instance_name = remote_context_.exec_config->remote_instance_name, .dispatch = *std::move(res), .cache_address = remote_context_.exec_config->cache_address, .platform_properties = std::move(platform_properties)}; } auto TargetService::ServeTarget( ::grpc::ServerContext* context, const ::justbuild::just_serve::ServeTargetRequest* request, ::justbuild::just_serve::ServeTargetResponse* response) -> ::grpc::Status { // acquire locks auto repo_lock = RepositoryGarbageCollector::SharedLock(*local_context_.storage_config); if (not repo_lock) { auto msg = std::string("Could not acquire repo gc SharedLock"); logger_->Emit(LogLevel::Error, msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } auto lock = GarbageCollector::SharedLock(*local_context_.storage_config); if (not lock) { auto msg = std::string("Could not acquire CAS gc SharedLock"); logger_->Emit(LogLevel::Error, msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } std::string download_error; std::vector required_digests; required_digests.reserve(request->required_digests_size()); for (int i = 0; i != request->required_digests_size(); ++i) { auto current = ArtifactDigestFactory::FromBazel( local_context_.storage_config->hash_function.GetType(), request->required_digests(i)); if (not current.has_value()) { download_error += fmt::format("Failed to convert a required digest {}\n", request->required_digests(i).hash()); continue; } bool const is_tree = current->IsTree(); required_digests.emplace_back() = Artifact::ObjectInfo{ .digest = *std::move(current), .type = is_tree ? ObjectType::Tree : ObjectType::File, }; } if (not required_digests.empty() and not apis_.remote->RetrieveToCas(required_digests, *apis_.local)) { download_error += "Failed to download all required artifacts\n"; } auto status = ServeTargetImpl(context, request, response); if (status.error_code() == grpc::StatusCode::OK or download_error.empty()) { return status; } logger_->Emit(LogLevel::Warning, download_error); return ::grpc::Status{ status.error_code(), fmt::format("{}{}", download_error, status.error_message())}; } auto TargetService::ServeTargetImpl( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetRequest* request, ::justbuild::just_serve::ServeTargetResponse* response) -> ::grpc::Status { // check target cache key hash for validity auto const target_cache_key_digest = ArtifactDigestFactory::FromBazel( local_context_.storage_config->hash_function.GetType(), request->target_cache_key_id()); if (not target_cache_key_digest) { logger_->Emit(LogLevel::Error, "{}", target_cache_key_digest.error()); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, target_cache_key_digest.error()}; } logger_->Emit( LogLevel::Debug, "ServeTarget({})", target_cache_key_digest->hash()); // set up the remote execution config instance for the orchestrated build auto remote_config = CreateRemoteExecutionConfig(request); if (not remote_config) { return std::move(remote_config).error(); } // get backend description auto description = BackendDescription::Describe(remote_config->remote_address, remote_config->platform_properties, remote_config->dispatch); if (not description) { auto err = fmt::format("Failed to create backend description:\n{}", description.error()); logger_->Emit(LogLevel::Error, err); return ::grpc::Status{::grpc::StatusCode::INTERNAL, err}; } // get a target cache instance with the correct computed shard auto const tc = local_context_.storage->TargetCache().WithShard(*description); auto const tc_key = TargetCacheKey{{*target_cache_key_digest, ObjectType::File}}; // check if target-level cache entry has already been computed if (auto target_entry = tc.Read(tc_key); target_entry) { // make sure all artifacts referenced in the target cache value are in // the remote cas std::vector artifacts; if (not target_entry->first.ToArtifacts(&artifacts)) { auto msg = fmt::format( "Failed to extract artifacts from target cache entry {}", target_entry->second.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } artifacts.emplace_back(target_entry->second); // add the tc value if (not apis_.local->RetrieveToCas(artifacts, *apis_.remote)) { auto msg = fmt::format( "Failed to upload to remote cas the artifacts referenced in " "the target cache entry {}", target_entry->second.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::UNAVAILABLE, msg}; } if (request->keep_artifact_root()) { auto keep = KeepRoot( target_entry->first, local_context_, apis_.local, lock_); if (keep) { logger_->Emit(LogLevel::Error, "{}", *keep); return ::grpc::Status{::grpc::StatusCode::INTERNAL, *keep}; } } // populate response with the target cache value (*response->mutable_target_value()) = ArtifactDigestFactory::ToBazel(target_entry->second.digest); return ::grpc::Status::OK; } // get target description from remote cas auto const& target_cache_key_info = Artifact::ObjectInfo{ .digest = *target_cache_key_digest, .type = ObjectType::File}; if (not apis_.local->IsAvailable(*target_cache_key_digest) and not apis_.remote->RetrieveToCas({target_cache_key_info}, *apis_.local)) { auto msg = fmt::format( "Could not retrieve blob {} from remote-execution endpoint", target_cache_key_info.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } auto const& target_description_str = apis_.local->RetrieveToMemory(target_cache_key_info); if (not target_description_str) { // this should not fail unless something really broke... auto msg = fmt::format( "Unexpected failure in retrieving blob {} from local CAS", target_cache_key_info.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } ExpressionPtr target_description_dict{}; try { target_description_dict = Expression::FromJson( nlohmann::json::parse(*target_description_str)); } catch (std::exception const& ex) { auto msg = fmt::format("Parsing TargetCacheKey {} failed with:\n{}", target_cache_key_digest->hash(), ex.what()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } if (not target_description_dict.IsNotNull() or not target_description_dict->IsMap()) { auto msg = fmt::format("TargetCacheKey {} should contain a map, but found {}", target_cache_key_digest->hash(), target_description_dict.ToJson().dump()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::NOT_FOUND, msg}; } std::string error_msg{}; // buffer to store various error messages // utility function to check the correctness of the TargetCacheKey auto check_key = [&target_description_dict, this, &target_cache_key_digest, &error_msg](std::string const& key) -> bool { if (not target_description_dict->At(key)) { error_msg = fmt::format("TargetCacheKey {} does not contain key \"{}\"", target_cache_key_digest->hash(), key); logger_->Emit(LogLevel::Warning, "{}", error_msg); return false; } return true; }; if (not check_key("repo_key") or not check_key("target_name") or not check_key("effective_config")) { return ::grpc::Status{::grpc::StatusCode::NOT_FOUND, error_msg}; } // get repository config blob path auto const& repo_key = target_description_dict->Get("repo_key", Expression::none_t{}); if (not repo_key.IsNotNull() or not repo_key->IsString()) { auto msg = fmt::format( "TargetCacheKey {}: \"repo_key\" value should be a string, but " "found {}", target_cache_key_digest->hash(), repo_key.ToJson().dump()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::NOT_FOUND, msg}; } auto const repo_key_dgst = ArtifactDigestFactory::Create(apis_.local->GetHashType(), repo_key->String(), 0, /*is_tree=*/false); if (not repo_key_dgst) { logger_->Emit(LogLevel::Error, "{}", repo_key_dgst.error()); return ::grpc::Status{::grpc::StatusCode::INTERNAL, repo_key_dgst.error()}; } if (not apis_.local->IsAvailable(*repo_key_dgst) and not apis_.remote->RetrieveToCas( {Artifact::ObjectInfo{.digest = *repo_key_dgst, .type = ObjectType::File}}, *apis_.local)) { auto msg = fmt::format( "Could not retrieve blob {} from remote-execution endpoint", repo_key->String()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } auto repo_config_path = local_context_.storage->CAS().BlobPath( *repo_key_dgst, /*is_executable=*/false); if (not repo_config_path) { // This should not fail unless something went really bad... auto msg = fmt::format( "Unexpected failure in retrieving blob {} from local CAS", repo_key->String()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } // populate the RepositoryConfig instance RepositoryConfig repository_config{}; std::string const main_repo{"0"}; // known predefined main repository name if (auto msg = DetermineRoots(serve_config_, local_context_.storage_config, main_repo, *repo_config_path, &repository_config, logger_)) { logger_->Emit(LogLevel::Info, "{}", *msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, *msg}; } // get the target name auto const& target_expr = target_description_dict->Get("target_name", Expression::none_t{}); if (not target_expr.IsNotNull() or not target_expr->IsString()) { auto msg = fmt::format( "TargetCacheKey {}: \"target_name\" value should be a string, but" " found {}", target_cache_key_digest->hash(), target_expr.ToJson().dump()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } auto target_name = nlohmann::json::object(); try { target_name = nlohmann::json::parse(target_expr->String()); } catch (std::exception const& ex) { auto msg = fmt::format( "TargetCacheKey {}: parsing \"target_name\" failed with:\n{}", target_cache_key_digest->hash(), ex.what()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } // get the effective config of the export target auto const& config_expr = target_description_dict->Get("effective_config", Expression::none_t{}); if (not config_expr.IsNotNull() or not config_expr->IsString()) { auto msg = fmt::format( "TargetCacheKey {}: \"effective_config\" value should be a string," " but found {}", target_cache_key_digest->hash(), config_expr.ToJson().dump()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } Configuration config{}; try { config = Configuration{ Expression::FromJson(nlohmann::json::parse(config_expr->String()))}; } catch (std::exception const& ex) { auto msg = fmt::format( "TargetCacheKey {}: parsing \"effective_config\" failed with:\n{}", target_cache_key_digest->hash(), ex.what()); logger_->Emit(LogLevel::Warning, "{}", msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, msg}; } // get the ConfiguredTarget auto entity = BuildMaps::Base::ParseEntityNameFromJson( target_name, BuildMaps::Base::EntityName{ BuildMaps::Base::NamedTarget{main_repo, ".", ""}}, &repository_config, [&error_msg, &target_name](std::string const& parse_err) { error_msg = fmt::format("Parsing target name {} failed with:\n {} ", target_name.dump(), parse_err); }); if (not entity) { logger_->Emit(LogLevel::Warning, "{}", error_msg); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, error_msg}; } auto configured_target = BuildMaps::Target::ConfiguredTarget{ .target = std::move(*entity), .config = std::move(config)}; // setup progress reporting; these instances need to be kept alive for // graph traversal, analysis, and build Statistics stats{}; Progress progress{}; // setup logging for analysis and build; write into a temporary file auto tmp_dir = local_context_.storage_config->CreateTypedTmpDir("serve-target"); if (not tmp_dir) { auto msg = std::string("Could not create TmpDir"); logger_->Emit(LogLevel::Error, msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } std::filesystem::path tmp_log{tmp_dir->GetPath() / "log"}; Logger logger{"serve-target", {LogSinkFile::CreateFactory(tmp_log)}}; AnalyseContext analyse_ctx{.repo_config = &repository_config, .storage = local_context_.storage, .statistics = &stats, .progress = &progress, .serve = serve_}; // analyse the configured target auto analyse_result = AnalyseTarget(&analyse_ctx, configured_target, serve_config_.jobs, std::nullopt /*request_action_input*/, &logger); if (not analyse_result) { // report failure locally, to keep track of it... auto msg = fmt::format("Failed to analyse target {}", configured_target.target.ToString()); logger_->Emit(LogLevel::Warning, "{}", msg); return HandleFailureLog(tmp_log, "analysis", response); } logger_->Emit( LogLevel::Info, "Analysed target {}", analyse_result->id.ToString()); auto jobs = serve_config_.build_jobs; if (jobs == 0) { jobs = serve_config_.jobs; } { // setup graph traverser GraphTraverser::CommandLineArguments traverser_args{}; traverser_args.jobs = jobs; traverser_args.build.timeout = serve_config_.action_timeout; traverser_args.stage = std::nullopt; traverser_args.rebuild = std::nullopt; // pack the remote context instances to be passed as needed RemoteContext const dispatch_context{ .auth = remote_context_.auth, .retry_config = remote_context_.retry_config, .exec_config = &(*remote_config)}; // Use a new ApiBundle that knows about local repository config and // dispatch endpoint for traversing auto const local_apis = ApiBundle::Create( &local_context_, &dispatch_context, &repository_config); ExecutionContext const exec_context{.repo_config = &repository_config, .apis = &local_apis, .remote_context = &dispatch_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const traverser{ std::move(traverser_args), &exec_context, ProgressReporter::Reporter(&stats, &progress, &logger), &logger}; // get the output artifacts auto const [artifacts, runfiles] = ReadOutputArtifacts(analyse_result->target); // get the analyse_result map outputs auto [actions, blobs, trees, tree_overlays] = analyse_result->result_map.ToResult(&stats, &progress, &logger); // collect cache targets and artifacts for target-level caching auto const cache_targets = analyse_result->result_map.CacheTargets(); auto cache_artifacts = CollectNonKnownArtifacts(cache_targets); // Clean up analyse_result map, now that it is no longer needed { TaskSystem ts{serve_config_.jobs}; analyse_result->result_map.Clear(&ts); } // perform build auto build_result = traverser.BuildAndStage(artifacts, runfiles, std::move(actions), std::move(blobs), std::move(trees), std::move(tree_overlays), std::move(cache_artifacts)); if (not build_result) { // report failure locally, to keep track of it... logger_->Emit(LogLevel::Warning, "Build for target {} failed", configured_target.target.ToString()); return HandleFailureLog(tmp_log, "build", response); } WriteTargetCacheEntries(cache_targets, build_result->extra_infos, jobs, local_apis, serve_config_.tc_strategy, tc, &logger, LogLevel::Error); if (build_result->failed_artifacts) { // report failure locally, to keep track of it... logger_->Emit( LogLevel::Warning, "Build result for target {} contains failed artifacts ", configured_target.target.ToString()); return HandleFailureLog(tmp_log, "artifacts", response); } } // now that the target cache key is in, make sure remote CAS has all // required entries if (auto target_entry = tc.Read(tc_key); target_entry) { // make sure all artifacts referenced in the target cache value are in // the remote cas std::vector tc_artifacts; if (not target_entry->first.ToArtifacts(&tc_artifacts)) { auto msg = fmt::format( "Failed to extract artifacts from target cache entry {}", target_entry->second.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } tc_artifacts.emplace_back(target_entry->second); // add the tc value if (not apis_.local->RetrieveToCas(tc_artifacts, *apis_.remote)) { auto msg = fmt::format( "Failed to upload to remote cas the artifacts referenced in " "the target cache entry {}", target_entry->second.ToString()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::UNAVAILABLE, msg}; } if (request->keep_artifact_root()) { auto keep_error = KeepRoot( target_entry->first, local_context_, apis_.local, lock_); if (keep_error) { logger_->Emit(LogLevel::Error, "{}", *keep_error); return ::grpc::Status{::grpc::StatusCode::INTERNAL, *keep_error}; } } // populate response with the target cache value (*response->mutable_target_value()) = ArtifactDigestFactory::ToBazel(target_entry->second.digest); return ::grpc::Status::OK; } // target cache value missing -- internally something is very wrong auto msg = fmt::format("Failed to read TargetCacheKey {} after store", target_cache_key_digest->hash()); logger_->Emit(LogLevel::Error, "{}", msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } auto TargetService::ServeTargetVariables( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetVariablesRequest* request, ::justbuild::just_serve::ServeTargetVariablesResponse* response) -> ::grpc::Status { auto const& root_tree{request->root_tree()}; auto const& target_file{request->target_file()}; auto const& target{request->target()}; logger_->Emit(LogLevel::Debug, "ServeTargetVariables({}, ...)", root_tree); // retrieve content of target file std::optional target_file_content{std::nullopt}; bool tree_found{false}; // Get repository lock before inspecting the root git cache auto repo_lock = RepositoryGarbageCollector::SharedLock(*local_context_.storage_config); if (not repo_lock) { auto msg = std::string("Could not acquire repo gc SharedLock"); logger_->Emit(LogLevel::Error, msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } // try in local build root Git cache if (auto res = GetBlobContent(local_context_.storage_config->GitRoot(), root_tree, target_file, logger_)) { tree_found = true; if (res->first) { if (not res->second) { // tree exists, but does not contain target file auto err = fmt::format( "Target-root {} found, but does not contain targets file " "{}", root_tree, target_file); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } target_file_content = *res->second; } } if (not target_file_content) { // try given extra repositories, in order for (auto const& path : serve_config_.known_repositories) { if (auto res = GetBlobContent(path, root_tree, target_file, logger_)) { tree_found = true; if (res->first) { if (not res->second) { // tree exists, but does not contain target file auto err = fmt::format( "Target-root {} found, but does not contain " "targets file {}", root_tree, target_file); return ::grpc::Status{ ::grpc::StatusCode::FAILED_PRECONDITION, err}; } target_file_content = *res->second; break; } } } } // report if failed to find root tree if (not target_file_content) { if (tree_found) { // something went wrong trying to read the target file blob auto err = fmt::format("Could not read targets file {}", target_file); return ::grpc::Status{::grpc::StatusCode::INTERNAL, err}; } // tree not found auto err = fmt::format("Missing target-root tree {}", root_tree); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // parse target file as json ExpressionPtr map{nullptr}; try { map = Expression::FromJson(nlohmann::json::parse(*target_file_content)); } catch (std::exception const& e) { auto err = fmt::format( "Failed to parse targets file {} as json with error:\n{}", target_file, e.what()); logger_->Emit(LogLevel::Error, err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (not map->IsMap()) { auto err = fmt::format("Targets file {} should contain a map, but found:\n{}", target_file, map->ToString()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // do validity checks (target present, is export, flexible_config valid) auto target_desc = map->At(target); if (not target_desc) { // target is not present auto err = fmt::format("Missing target {} in targets file {}", nlohmann::json(target).dump(), target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } auto export_desc = target_desc->get()->At("type"); if (not export_desc) { auto err = fmt::format( "Missing \"type\" field for target {} in targets file {}.", nlohmann::json(target).dump(), target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (not export_desc->get()->IsString()) { auto err = fmt::format( "Expected field \"type\" for target {} in targets file {} to be a " "string, but found:\n{}", nlohmann::json(target).dump(), target_file, export_desc->get()->ToString()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (export_desc->get()->String() != "export") { // target is not of "type" : "export" auto err = fmt::format(R"(target {} is not of "type" : "export")", nlohmann::json(target).dump()); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } auto flexible_config = target_desc->get()->At("flexible_config"); if (not flexible_config) { // respond with success and an empty flexible_config list return ::grpc::Status::OK; } if (not flexible_config->get()->IsList()) { auto err = fmt::format( "Field \"flexible_config\" for target {} in targets file {} should " "be a list, but found {}", nlohmann::json(target).dump(), target_file, flexible_config->get()->ToString()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // populate response with flexible_config list auto flexible_config_list = flexible_config->get()->List(); for (auto const& elem : flexible_config_list) { if (not elem->IsString()) { auto err = fmt::format( "Field \"flexible_config\" for target {} in targets file {} " "has non-string entry {}", nlohmann::json(target).dump(), target_file, elem->ToString()); logger_->Emit(LogLevel::Error, "{}", err); response->clear_flexible_config(); // must be unset return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } response->add_flexible_config(elem->String()); } // respond with success return ::grpc::Status::OK; } auto TargetService::ServeTargetDescription( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetDescriptionRequest* request, ::justbuild::just_serve::ServeTargetDescriptionResponse* response) -> ::grpc::Status { auto const& root_tree{request->root_tree()}; auto const& target_file{request->target_file()}; auto const& target{request->target()}; logger_->Emit( LogLevel::Debug, "ServeTargetDescription({}, ...)", root_tree); // retrieve content of target file std::optional target_file_content{std::nullopt}; bool tree_found{false}; // Get repository lock before inspecting the root git cache auto repo_lock = RepositoryGarbageCollector::SharedLock(*local_context_.storage_config); if (not repo_lock) { auto msg = std::string("Could not acquire repo gc SharedLock"); logger_->Emit(LogLevel::Error, msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, msg}; } // try in local build root Git cache if (auto res = GetBlobContent(local_context_.storage_config->GitRoot(), root_tree, target_file, logger_)) { tree_found = true; if (res->first) { if (not res->second) { // tree exists, but does not contain target file auto err = fmt::format( "Target-root {} found, but does not contain targets file " "{}", root_tree, target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } target_file_content = *res->second; } } if (not target_file_content) { // try given extra repositories, in order for (auto const& path : serve_config_.known_repositories) { if (auto res = GetBlobContent(path, root_tree, target_file, logger_)) { tree_found = true; if (res->first) { if (not res->second) { // tree exists, but does not contain target file auto err = fmt::format( "Target-root {} found, but does not contain " "targets file {}", root_tree, target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{ ::grpc::StatusCode::FAILED_PRECONDITION, err}; } target_file_content = *res->second; break; } } } } // report if failed to find root tree if (not target_file_content) { if (tree_found) { // something went wrong trying to read the target file blob auto err = fmt::format("Could not read targets file {}", target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::INTERNAL, err}; } // tree not found auto err = fmt::format("Missing target-root tree {}", root_tree); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // parse target file as json ExpressionPtr map{nullptr}; try { map = Expression::FromJson(nlohmann::json::parse(*target_file_content)); } catch (std::exception const& e) { auto err = fmt::format( "Failed to parse targets file {} as json with error:\n{}", target_file, e.what()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (not map->IsMap()) { auto err = fmt::format("Targets file {} should contain a map, but found:\n{}", target_file, map->ToString()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // do validity checks (target is present and is of "type": "export") auto target_desc = map->At(target); if (not target_desc) { // target is not present auto err = fmt::format("Missing target {} in targets file {}", nlohmann::json(target).dump(), target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } auto export_desc = target_desc->get()->At("type"); if (not export_desc) { auto err = fmt::format( "Missing \"type\" field for target {} in targets file {}.", nlohmann::json(target).dump(), target_file); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (not export_desc->get()->IsString()) { auto err = fmt::format( "Expected field \"type\" for target {} in targets file {} to be a " "string, but found:\n{}", nlohmann::json(target).dump(), target_file, export_desc->get()->ToString()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } if (export_desc->get()->String() != "export") { // target is not of "type" : "export" auto err = fmt::format(R"(target {} is not of "type" : "export")", nlohmann::json(target).dump()); logger_->Emit(LogLevel::Error, "{}", err); return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; } // populate response description object with fields as-is auto desc = nlohmann::json::object(); if (auto doc = target_desc->get()->Get("doc", Expression::none_t{}); doc.IsNotNull()) { desc["doc"] = doc->ToJson(); } if (auto config_doc = target_desc->get()->Get("config_doc", Expression::none_t{}); config_doc.IsNotNull()) { desc["config_doc"] = config_doc->ToJson(); } if (auto flexible_config = target_desc->get()->Get("flexible_config", Expression::none_t{}); flexible_config.IsNotNull()) { desc["flexible_config"] = flexible_config->ToJson(); } // acquire lock for CAS auto lock = GarbageCollector::SharedLock(*local_context_.storage_config); if (not lock) { auto error_msg = fmt::format("Could not acquire gc SharedLock"); logger_->Emit(LogLevel::Error, error_msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, error_msg}; } // store description blob to local CAS and sync with remote CAS; // we keep the documentation strings as close to actual as possible, so we // do not fail if they contain invalid UTF-8 characters, instead we use the // UTF-8 replacement character to solve any decoding errors. std::string description_str; try { description_str = desc.dump(2, ' ', false, nlohmann::json::error_handler_t::replace); } catch (std::exception const& ex) { // normally shouldn't happen std::string err{"Failed to dump backend JSON description to string"}; logger_->Emit(LogLevel::Error, err); return ::grpc::Status{::grpc::StatusCode::INTERNAL, err}; } if (auto dgst = local_context_.storage->CAS().StoreBlob(description_str, /*is_executable=*/false)) { if (not apis_.local->RetrieveToCas( {Artifact::ObjectInfo{.digest = *dgst, .type = ObjectType::File}}, *apis_.remote)) { auto error_msg = fmt::format( "Failed to upload to remote cas the description blob {}", dgst->hash()); logger_->Emit(LogLevel::Error, "{}", error_msg); return ::grpc::Status{::grpc::StatusCode::UNAVAILABLE, error_msg}; } // populate response (*response->mutable_description_id()) = ArtifactDigestFactory::ToBazel(*dgst); return ::grpc::Status::OK; } // failed to store blob const auto* const error_msg = "Failed to store description blob to local cas"; logger_->Emit(LogLevel::Error, error_msg); return ::grpc::Status{::grpc::StatusCode::INTERNAL, error_msg}; } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/target.hpp000066400000000000000000000176361516554100600312400ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP #define INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" #include "justbuild/just_serve/just_serve.pb.h" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/utils/cpp/expected.hpp" // The target-level cache service. class TargetService final : public justbuild::just_serve::Target::Service { public: explicit TargetService( gsl::not_null const& serve_config, gsl::not_null const& local_context, gsl::not_null const& remote_context, gsl::not_null const& apis, gsl::not_null const& lock, ServeApi const* serve = nullptr) noexcept : serve_config_{*serve_config}, local_context_{*local_context}, remote_context_{*remote_context}, apis_{*apis}, lock_{lock}, serve_{serve} {} // Given a target-level caching key, returns the computed value. In doing // so, it can build on the associated endpoint passing the // RemoteExecutionProperties contained in the ServeTargetRequest. // The execution backend description, the resulting target cache value, // and all other artifacts referenced therein MUST be made available in // the CAS of the associated remote-execution endpoint. // // A failure to analyse or build a known target (i.e., a target for which // we have all the needed information available) should NOT be reported as // an error. Instead, the failure log should be uploaded as a blob to the // CAS of the associated remote-execution endpoint and its digest provided // to the client in the response field `log`. In this case, the field // `target_value` MUST not be set. // // If the status has a code different from `OK` or `NOT_FOUND`, the // response MUST not be used. // // Errors: // * `NOT_FOUND`: Unknown target or missing needed local information. // This should only be used for non-fatal failures. // * `FAILED_PRECONDITION`: Required entries missing in the remote // execution endpoint. // * `UNAVAILABLE`: Could not communicate with the remote-execution // endpoint. // * `INVALID_ARGUMENT`: The client provided invalid arguments in request. // * `INTERNAL`: Internally, something is very broken. auto ServeTarget( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetRequest* /*request*/, ::justbuild::just_serve::ServeTargetResponse* /*response*/) -> ::grpc::Status override; // Given the target-level root tree and the name of an export target, // returns the list of flexible variables from that target's description. // // If the status has a code different from `OK`, the response MUST not be // used. // // Errors: // * `FAILED_PRECONDITION`: An error occurred in retrieving the // configuration of the requested target, such as missing entries // (target-root, target file, target name), unparsable target file, or // requested target not being of "type" : "export". // * `INTERNAL`: Internally, something is very broken. auto ServeTargetVariables( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetVariablesRequest* request, ::justbuild::just_serve::ServeTargetVariablesResponse* response) -> ::grpc::Status override; // Given the target-level root tree and the name of an export target, // returns the digest of the blob containing the flexible variables field, // as well as the documentation fields pertaining tho the target and // its configuration variables, as taken from the target's description. // This information should be enough for a client to produce locally a // full description of said target. // // The server MUST make the returned blob available in the remote CAS. // // If the status has a code different from `OK`, the response MUST not be // used. // // Errors: // * `FAILED_PRECONDITION`: An error occurred in retrieving the // configuration of the requested target, such as missing entries // (target-root, target file, target name), unparsable target file, or // requested target not being of "type" : "export". // * `UNAVAILABLE`: Could not communicate with the remote CAS. // * `INTERNAL`: Internally, something is very broken. auto ServeTargetDescription( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetDescriptionRequest* request, ::justbuild::just_serve::ServeTargetDescriptionResponse* response) -> ::grpc::Status override; private: RemoteServeConfig const& serve_config_; LocalContext const& local_context_; RemoteContext const& remote_context_; ApiBundle const& apis_; gsl::not_null lock_; ServeApi const* const serve_ = nullptr; std::shared_ptr logger_{std::make_shared("target-service")}; auto ServeTargetImpl( ::grpc::ServerContext* /*context*/, const ::justbuild::just_serve::ServeTargetRequest* /*request*/, ::justbuild::just_serve::ServeTargetResponse* /*response*/) -> ::grpc::Status; /// \brief Get from remote and parse the endpoint configuration. The method /// also ensures the content has the expected format. /// \returns The dispatch list stored as JSON object on success or an /// unexpected error as grpc::Status. [[nodiscard]] auto GetDispatchList( ArtifactDigest const& dispatch_digest) noexcept -> expected, ::grpc::Status>; /// \brief Handles the processing of the log after a failed analysis or /// build. Will populate the response as needed and set the status to be /// returned to the client. /// \param failure_scope String stating where the failure occurred, to be /// included in the local error messaging. [[nodiscard]] auto HandleFailureLog( std::filesystem::path const& logfile, std::string const& failure_scope, ::justbuild::just_serve::ServeTargetResponse* response) noexcept -> ::grpc::Status; /// \brief Create the execution configuration needed to shard the target /// cache and dispatch the build to the remote endpoint. /// \returns A set up RemoteExecutionConfig or an unexpected error as /// grpc::Status. [[nodiscard]] auto CreateRemoteExecutionConfig( const ::justbuild::just_serve::ServeTargetRequest* request) noexcept -> expected; }; #endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/target_utils.cpp000066400000000000000000000262501516554100600324430ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/serve_api/serve_service/target_utils.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/utils/cpp/expected.hpp" auto GetServingRepository(RemoteServeConfig const& serve_config, StorageConfig const& storage_config, std::string const& tree_id, std::shared_ptr const& logger) -> std::optional { // try the Git cache repository auto in_git_cas = GitRepo::IsTreeInRepo(storage_config.GitRoot(), tree_id); if (not in_git_cas.has_value()) { logger->Emit(LogLevel::Info, "ServeTarget: While checking existence of " "tree {} in repository {}:\n{}", tree_id, storage_config.GitRoot().string(), std::move(in_git_cas).error()); } else if (*in_git_cas) { return storage_config.GitRoot(); } // check the known repositories for (auto const& path : serve_config.known_repositories) { auto in_repo = GitRepo::IsTreeInRepo(path, tree_id); if (not in_repo.has_value()) { logger->Emit(LogLevel::Info, "ServeTarget: While checking existence of " "tree {} in repository {}:\n{}", tree_id, path.string(), std::move(in_repo).error()); } else if (*in_repo) { return path; } } return std::nullopt; // tree cannot be served } auto DetermineRoots(RemoteServeConfig const& serve_config, gsl::not_null storage_config, std::string const& main_repo, std::filesystem::path const& repo_config_path, gsl::not_null const& repository_config, std::shared_ptr const& logger) -> std::optional { // parse repository configuration file auto repos = nlohmann::json::object(); try { std::ifstream fs(repo_config_path); repos = nlohmann::json::parse(fs); if (not repos.is_object()) { return fmt::format( "Repository configuration file {} does not contain a map.", repo_config_path.string()); } } catch (std::exception const& ex) { return fmt::format("Parsing repository config file {} failed with:\n{}", repo_config_path.string(), ex.what()); } if (not repos.contains(main_repo)) { return fmt::format( "Repository configuration does not contain expected main " "repository {}", main_repo); } // populate RepositoryConfig instance for (auto const& [repo, desc] : repos.items()) { // root parser auto parse_keyword_root = [&serve_config, storage_config, &desc = desc, &repo = repo, logger]( std::string const& keyword) -> expected { auto it = desc.find(keyword); if (it != desc.end()) { auto parsed_root = FileRoot::ParseRoot(storage_config, repo, keyword, *it); if (not parsed_root) { return unexpected{std::move(parsed_root).error()}; } // check that root has absent-like format if (not parsed_root->first.IsAbsent()) { return unexpected{fmt::format( "Expected {} to have absent Git tree format, but " "found {}", keyword, it->dump())}; } // find the serving repository for the root tree auto tree_id = *parsed_root->first.GetAbsentTreeId(); auto repo_path = GetServingRepository( serve_config, *storage_config, tree_id, logger); if (not repo_path) { return unexpected{fmt::format( "{} tree {} is not known", keyword, tree_id)}; } // set the root as present if (auto root = FileRoot::FromGit(storage_config, *repo_path, tree_id, parsed_root->first.IgnoreSpecial())) { return *std::move(root); } } return unexpected{ fmt::format("Missing {} for repository {}", keyword, repo)}; }; auto ws_root = parse_keyword_root("workspace_root"); if (not ws_root) { return std::move(ws_root).error(); } auto info = RepositoryConfig::RepositoryInfo{*std::move(ws_root)}; if (auto target_root = parse_keyword_root("target_root")) { info.target_root = *std::move(target_root); } else { return std::move(target_root).error(); } if (auto rule_root = parse_keyword_root("rule_root")) { info.rule_root = *std::move(rule_root); } else { return std::move(rule_root).error(); } if (auto expression_root = parse_keyword_root("expression_root")) { info.expression_root = *std::move(expression_root); } else { return std::move(expression_root).error(); } auto it_bindings = desc.find("bindings"); if (it_bindings != desc.end()) { if (not it_bindings->is_object()) { return fmt::format( "bindings has to be a string-string map, but found {}", it_bindings->dump()); } for (auto const& [local_name, global_name] : it_bindings->items()) { if (not repos.contains(global_name)) { return fmt::format( "Binding {} for {} in {} does not refer to a " "defined repository.", global_name, local_name, repo); } info.name_mapping[local_name] = global_name; } } else { return fmt::format("Missing bindings for repository {}", repo); } auto parse_keyword_file_name = [&desc = desc, &repo = repo]( std::string* keyword_file_name, std::string const& keyword) -> expected { auto it = desc.find(keyword); if (it != desc.end()) { *keyword_file_name = *it; return std::monostate{}; } return unexpected{ fmt::format("Missing {} for repository {}", keyword, repo)}; }; if (auto result = parse_keyword_file_name(&info.target_file_name, "target_file_name"); not result) { return std::move(result).error(); } if (auto result = parse_keyword_file_name(&info.rule_file_name, "rule_file_name"); not result) { return std::move(result).error(); } if (auto result = parse_keyword_file_name(&info.expression_file_name, "expression_file_name"); not result) { return std::move(result).error(); } repository_config->SetInfo(repo, std::move(info)); } // success return std::nullopt; } auto GetBlobContent(std::filesystem::path const& repo_path, std::string const& tree_id, std::string const& rel_path, std::shared_ptr const& logger) -> std::optional>> { if (auto git_cas = GitCAS::Open(repo_path)) { if (auto repo = GitRepo::Open(git_cas)) { // check if tree exists auto wrapped_logger = std::make_shared( [logger, repo_path, tree_id](auto const& msg, bool fatal) { if (fatal) { logger->Emit(LogLevel::Debug, "ServeTargetVariables: While checking if " "tree {} exists in repository {}:\n{}", tree_id, repo_path.string(), msg); } }); if (repo->CheckTreeExists(tree_id, wrapped_logger) == true) { // get tree entry by path if (auto entry_info = repo->GetObjectByPathFromTree(tree_id, rel_path)) { wrapped_logger = std::make_shared( [logger, repo_path, blob_id = entry_info->id]( auto const& msg, bool fatal) { if (fatal) { logger->Emit( LogLevel::Debug, "ServeTargetVariables: While retrieving " "blob {} from repository {}:\n{}", blob_id, repo_path.string(), msg); } }); // get blob content return repo->TryReadBlob(entry_info->id, wrapped_logger); } // trace failure to get entry logger->Emit(LogLevel::Debug, "ServeTargetVariables: Failed to retrieve entry " "{} in tree {} from repository {}", rel_path, tree_id, repo_path.string()); return std::pair(false, std::nullopt); // could not read blob } } } return std::nullopt; // tree not found or errors while retrieving tree } just-buildsystem-justbuild-b1fb5fa/src/buildtool/serve_api/serve_service/target_utils.hpp000066400000000000000000000057161516554100600324540ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_UTILS_HPP #define INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_UTILS_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/storage/config.hpp" // Methods used by ServeTarget remote service /// \brief Check if tree exists in the given repository. [[nodiscard]] auto IsTreeInRepo(std::string const& tree_id, std::filesystem::path const& repo_path, std::shared_ptr const& logger) -> bool; /// \brief For a given tree id, find the known repository that can serve it. [[nodiscard]] auto GetServingRepository(RemoteServeConfig const& serve_config, StorageConfig const& storage_config, std::string const& tree_id, std::shared_ptr const& logger) -> std::optional; /// \brief Parse the stored repository configuration blob and populate the /// RepositoryConfig instance. /// \returns nullopt on success, error message as a string otherwise. [[nodiscard]] auto DetermineRoots( RemoteServeConfig const& serve_config, gsl::not_null storage_config, std::string const& main_repo, std::filesystem::path const& repo_config_path, gsl::not_null const& repository_config, std::shared_ptr const& logger) -> std::optional; // Methods used by ServeTargetVariables remote service /// \brief Get the blob content at given path inside a Git tree. /// \returns If tree found, pair of "no-internal-errors" flag and content of /// blob at the path specified if blob exists, nullopt otherwise. [[nodiscard]] auto GetBlobContent(std::filesystem::path const& repo_path, std::string const& tree_id, std::string const& rel_path, std::shared_ptr const& logger) -> std::optional>>; #endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/000077500000000000000000000000001516554100600240475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/TARGETS000066400000000000000000000134201516554100600251030ustar00rootroot00000000000000{ "config": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "hdrs": ["config.hpp"] , "deps": [ "backend_description" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "storage"] } , "repository_garbage_collector": { "type": ["@", "rules", "CC", "library"] , "name": ["repository_garbage_collector"] , "hdrs": ["repository_garbage_collector.hpp"] , "srcs": ["repository_garbage_collector.cpp"] , "deps": ["config", ["src/utils/cpp", "file_locking"]] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "storage"] } , "storage": { "type": ["@", "rules", "CC", "library"] , "name": ["storage"] , "hdrs": [ "storage.hpp" , "local_cas.hpp" , "local_cas.tpp" , "local_ac.hpp" , "local_ac.tpp" , "target_cache.hpp" , "target_cache.tpp" , "target_cache_key.hpp" , "target_cache_entry.hpp" , "large_object_cas.hpp" , "large_object_cas.tpp" , "uplinker.hpp" ] , "srcs": ["target_cache_entry.cpp", "uplinker.cpp"] , "deps": [ "backend_description" , "config" , "file_chunker" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_storage"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_cas"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "storage"] , "private-deps": [["src/buildtool/build_engine/expression", "expression_ptr_interface"]] } , "garbage_collector": { "type": ["@", "rules", "CC", "library"] , "name": ["garbage_collector"] , "hdrs": ["garbage_collector.hpp"] , "srcs": ["garbage_collector.cpp"] , "deps": ["config", ["src/utils/cpp", "file_locking"]] , "private-deps": [ "compactifier" , "storage" , ["@", "fmt", "", "fmt"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/execution_api/common", "message_limits"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "storage"] } , "compactifier": { "type": ["@", "rules", "CC", "library"] , "name": ["compactifier"] , "hdrs": ["compactifier.hpp"] , "private-hdrs": ["compactification_task.hpp"] , "srcs": ["compactifier.cpp", "compactification_task.cpp"] , "deps": ["storage"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/file_system", "file_storage"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "buildtool", "storage"] } , "fs_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["fs_utils"] , "hdrs": ["fs_utils.hpp"] , "srcs": ["fs_utils.cpp"] , "deps": [ "config" , "storage" , ["src/buildtool/common", "user_structs"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system/symlinks", "pragma_special"] ] , "stage": ["src", "buildtool", "storage"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "tmp_dir"] ] } , "file_chunker": { "type": ["@", "rules", "CC", "library"] , "name": ["file_chunker"] , "hdrs": ["file_chunker.hpp"] , "srcs": ["file_chunker.cpp"] , "stage": ["src", "buildtool", "storage"] , "private-deps": [["@", "gsl", "", "gsl"]] } , "backend_description": { "type": ["@", "rules", "CC", "library"] , "name": ["backend_description"] , "hdrs": ["backend_description.hpp"] , "srcs": ["backend_description.cpp"] , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "storage"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/file_system", "object_type"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/backend_description.cpp000066400000000000000000000075501516554100600305540ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/backend_description.hpp" #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/object_type.hpp" BackendDescription::BackendDescription() noexcept { if (auto dummy = Describe( /*address=*/std::nullopt, /*properties=*/{}, /*dispatch=*/{})) { description_ = dummy->description_; } else { description_ = std::make_shared(); } } auto BackendDescription::Describe( std::optional const& address, ExecutionProperties const& properties, std::vector const& dispatch) noexcept -> expected { nlohmann::json description; try { description["remote_address"] = address ? address->ToJson() : nlohmann::json{}; description["platform_properties"] = properties; } catch (std::exception const& e) { return unexpected{ fmt::format("Failed to serialize remote address and " "platform_properties:\n{}", e.what())}; } if (not dispatch.empty()) { try { // only add the dispatch list, if not empty, so that keys remain // not only more readable, but also backwards compatible with // earlier versions. auto dispatch_list = nlohmann::json::array(); for (auto const& [props, endpoint] : dispatch) { auto entry = nlohmann::json::array(); entry.push_back(nlohmann::json(props)); entry.push_back(endpoint.ToJson()); dispatch_list.push_back(entry); } description["endpoint dispatch list"] = dispatch_list; } catch (std::exception const& e) { return unexpected{fmt::format( "Failed to serialize endpoint dispatch list:\n{}", e.what())}; } } std::shared_ptr result; std::shared_ptr sha256; try { // json::dump with json::error_handler_t::replace will not throw an // exception if invalid UTF-8 sequences are detected in the input. // Instead, it will replace them with the UTF-8 replacement // character, but still it needs to be inside a try-catch clause to // ensure the noexcept modifier of the enclosing function. result = std::make_shared(description.dump( 2, ' ', false, nlohmann::json::error_handler_t::replace)); std::string hash = ArtifactDigestFactory::HashDataAs( HashFunction(HashFunction::Type::PlainSHA256), *result) .hash(); sha256 = std::make_shared(std::move(hash)); } catch (std::exception const& e) { return unexpected{fmt::format( "Failed to dump backend description to JSON:\n{}", e.what())}; } return BackendDescription(std::move(result), std::move(sha256)); } auto BackendDescription::HashContent(HashFunction hash_function) const noexcept -> ArtifactDigest { return ArtifactDigestFactory::HashDataAs( hash_function, GetDescription()); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/backend_description.hpp000066400000000000000000000043211516554100600305520ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_BACKEND_DESCRIPTION_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_BACKEND_DESCRIPTION_HPP #include #include #include #include #include #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/expected.hpp" class BackendDescription final { public: explicit BackendDescription() noexcept; /// \brief String representation of the used execution backend. [[nodiscard]] static auto Describe( std::optional const& address, ExecutionProperties const& properties, std::vector const& dispatch) noexcept -> expected; [[nodiscard]] auto GetDescription() const noexcept -> std::string const& { return *description_; } [[nodiscard]] auto HashContent(HashFunction hash_function) const noexcept -> ArtifactDigest; [[nodiscard]] auto operator==( BackendDescription const& other) const noexcept -> bool { return sha256_ == other.sha256_ or *sha256_ == *other.sha256_; } private: explicit BackendDescription(std::shared_ptr description, std::shared_ptr sha256) noexcept : description_{std::move(description)}, sha256_{std::move(sha256)} {} std::shared_ptr description_; std::shared_ptr sha256_; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_BACKEND_DESCRIPTION_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/compactification_task.cpp000066400000000000000000000127301516554100600311140ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/compactification_task.hpp" #include #include #include #include #include #include //std::move #include #include "gsl/gsl" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/utils/cpp/path_hash.hpp" namespace { [[nodiscard]] auto GetObjectTask(CompactificationTask const& task, ObjectType type) noexcept -> CompactificationTask::ObjectTask const&; [[nodiscard]] auto GetFilterTypes(CompactificationTask const& task) -> std::vector; using FilterResult = std::optional>; [[nodiscard]] auto FilterEntries(CompactificationTask const& task, ObjectType type) noexcept -> FilterResult; } // namespace [[nodiscard]] auto CompactifyConcurrently( CompactificationTask const& task) noexcept -> bool { std::atomic_bool failed = false; std::unordered_map scan_results; { TaskSystem ts; // Filter entries to create execution tasks: try { for (auto type : GetFilterTypes(task)) { auto tstask = [result = &scan_results[type], &failed, type, &task] { *result = ::FilterEntries(task, type); if (not *result) { failed = true; } }; ts.QueueTask(std::move(tstask)); } } catch (...) { ts.Shutdown(); return false; } } // Init compactification tasks: if (not failed) { TaskSystem ts; for (auto const& [type, subtasks] : scan_results) { auto const& task_callback = GetObjectTask(task, type); for (auto const& entry : *subtasks) { try { auto tstask = [&failed, &task, &task_callback, &entry] { if (not failed and not std::invoke(task_callback, task, entry)) { failed = true; } }; ts.QueueTask(std::move(tstask)); } catch (...) { ts.Shutdown(); return false; } } } } return not failed; } namespace { [[nodiscard]] auto GetObjectTask(CompactificationTask const& task, ObjectType type) noexcept -> CompactificationTask::ObjectTask const& { switch (type) { case ObjectType::Symlink: case ObjectType::File: return task.f_task; case ObjectType::Executable: return task.x_task; case ObjectType::Tree: return task.t_task; } Ensures(false); // unreachable } [[nodiscard]] auto GetFilterTypes(CompactificationTask const& task) -> std::vector { static constexpr std::array kObjectTypes{ ObjectType::File, ObjectType::Tree, ObjectType::Executable}; // Ensure that types point to unique disk locations. // Duplication of roots leads to duplication of tasks. std::vector result; std::unordered_set unique_roots; for (ObjectType type : kObjectTypes) { auto root = task.cas.StorageRoot(type, task.large); if (unique_roots.insert(std::move(root)).second) { result.emplace_back(type); } } return result; } [[nodiscard]] auto FilterEntries(CompactificationTask const& task, ObjectType type) noexcept -> FilterResult { std::vector result; auto const& storage_root = task.cas.StorageRoot(type, task.large); // Check there are entries to process: if (not FileSystemManager::IsDirectory(storage_root)) { return result; } FileSystemManager::UseDirEntryFunc callback = [&task, &storage_root, &result](std::filesystem::path const& entry, bool /*unused*/) -> bool { // Filter entries: try { if (std::invoke(task.filter, storage_root / entry)) { result.push_back(entry); } } catch (...) { return false; } return true; }; // Read the ObjectType storage directory: if (not FileSystemManager::ReadDirectoryEntriesRecursive(storage_root, callback)) { result.clear(); task.Log(LogLevel::Error, "Scanning: Failed to read {}", storage_root.string()); return std::nullopt; } return result; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/compactification_task.hpp000066400000000000000000000076411516554100600311260ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFICATION_TASK_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFICATION_TASK_HPP #include #include #include #include "fmt/core.h" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/local_cas.hpp" /// \brief Set of data that defines a compactification task. /// \param cas CAS to be scanned. /// \param large True if large storages need to be scanned. /// \param f_task A handler for ObjectType::File entries. /// \param x_task A handler for ObjectType::Executable entries. /// It is never called during scanning of large storages. /// \param t_task A handler for ObjectType::Tree entries. /// \param logger A callback for logging. /// \param filter A callback that defines which files must be processed. struct CompactificationTask final { using Logger = std::function; using Filter = std::function; using ObjectTask = std::function; static inline const Logger kLoggerDefault = [](LogLevel level, std::string const& message) { ::Logger::Log(level, "{}", message); }; static inline const Filter kFilterDefault = [](std::filesystem::path const& /*unused*/) { return false; }; static inline const ObjectTask kObjectTaskDefault = [](CompactificationTask const& /*unused*/, std::filesystem::path const& /*unused*/) { kLoggerDefault(LogLevel::Error, "Default ObjectTask was called"); return false; }; LocalCAS const& cas; bool large = false; Logger logger = kLoggerDefault; Filter filter = kFilterDefault; ObjectTask f_task = kObjectTaskDefault; ObjectTask x_task = kObjectTaskDefault; ObjectTask t_task = kObjectTaskDefault; /// \brief Log a formatted error. template void Log(LogLevel level, std::string const& message, Args&&... args) const noexcept; }; /// \brief Execute the comapctification task using multiple threads. /// Execution of the CompactificationTask-defined logic begins only after the /// entire storage has been scanned, otherwise the storage may be invalidated. /// Example: casf is scanned while another thread puts new split chunks there. /// \return True if execution was successful. [[nodiscard]] auto CompactifyConcurrently( CompactificationTask const& task) noexcept -> bool; template void CompactificationTask::Log( LogLevel level, std::string const& message, // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) Args&&... args) const noexcept { if (not logger) { ::Logger::Log(LogLevel::Error, "Logger is missing."); return; } auto msg = fmt::vformat(message, fmt::make_format_args(args...)); try { std::invoke(logger, level, msg); } catch (...) { ::Logger::Log(LogLevel::Error, "Failed to invoke a logger"); ::Logger::Log(level, "{}", msg); } } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFICATION_TASK_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/compactifier.cpp000066400000000000000000000301651516554100600272250ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/compactifier.hpp" #include #include #include #include #include #include #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/compactification_task.hpp" #include "src/utils/cpp/hex_string.hpp" namespace { /// \brief Remove invalid entries from the key directory. The directory itself /// can be removed too, if it has an invalid name. /// A task is keyed by a two-letter directory name and the type of a storage /// being checked. /// \tparam kType Type of the storage to inspect. /// \param task Owning compactification task. /// \param entry Directory entry. /// \return True if the entry directory doesn't contain invalid /// entries. template [[nodiscard]] auto RemoveInvalid(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool; /// \brief Remove spliced entries from the kType storage. /// A task is keyed by a directory name concisting of two letters and kType... /// storages need to be checked. /// \tparam kLargeType Type of the large storage to scan. /// \tparam kType Types of the storages to inspect. /// \param task Owning compactification task. /// \param entry Directory entry. /// \return True if the entry directory doesn't contain spliced /// entries. template requires(sizeof...(kType) != 0) [[nodiscard]] auto RemoveSpliced(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool; /// \brief Split and remove a key entry from the kType storage. Results of /// splitting are added to the LocalCAS. /// \tparam kType Type of the storage to inspect. /// \param task Owning compactification task. /// \param entry File entry. /// \return True if the file was split succesfully. template [[nodiscard]] auto SplitLarge(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool; } // namespace auto Compactifier::RemoveInvalid(LocalCAS const& cas) noexcept -> bool { auto logger = [](LogLevel level, std::string const& msg) { Logger::Log( level, "Compactification: Removal of invalid files:\n{}", msg); }; // Collect directories and remove from the storage files and directories // having invalid names. // In general, the number of files in the storage is not limited, thus // parallelization is done using subdirectories to not run out of memory. CompactificationTask task{.cas = cas, .large = false, .logger = logger, .filter = FileSystemManager::IsDirectory, .f_task = ::RemoveInvalid, .x_task = ::RemoveInvalid, .t_task = ::RemoveInvalid}; return CompactifyConcurrently(task); } auto Compactifier::RemoveSpliced(LocalCAS const& cas) noexcept -> bool { auto logger = [](LogLevel level, std::string const& msg) { Logger::Log( level, "Compactification: Removal of spliced files:\n{}", msg); }; // Collect directories from large storages and remove from the storage files // having correponding large entries. // In general, the number of files in the storage is not limited, thus // parallelization is done using subdirectories to not run out of memory. CompactificationTask task{ .cas = cas, .large = true, .logger = logger, .filter = FileSystemManager::IsDirectory, .f_task = ::RemoveSpliced, .t_task = ::RemoveSpliced}; return CompactifyConcurrently(task); } auto Compactifier::SplitLarge(LocalCAS const& cas, size_t threshold) noexcept -> bool { auto logger = [](LogLevel level, std::string const& msg) { Logger::Log(level, "Compactification: Splitting:\n{}", msg); }; // Collect files larger than the threshold and split them. // Concurrently scanning a directory and putting new entries there may cause // the scan to fail. To avoid that, parallelization is done using // files, although this may result in a run out of memory. CompactificationTask task{ .cas = cas, .large = false, .logger = logger, .filter = [threshold](std::filesystem::path const& path) { return not FileSystemManager::IsDirectory(path) and std::filesystem::file_size(path) >= threshold; }, .f_task = ::SplitLarge, .x_task = ::SplitLarge, .t_task = ::SplitLarge}; return CompactifyConcurrently(task); } namespace { template [[nodiscard]] auto RemoveInvalid(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool { auto const directory = task.cas.StorageRoot(kType) / key; // Check there are entries to process: if (not FileSystemManager::IsDirectory(directory)) { return true; } // Calculate reference hash size: auto const hash_size = task.cas.GetHashFunction().MakeHasher().GetHashLength(); auto const file_name_size = hash_size - FileStorageData::kDirectoryNameLength; // Check the directory itself is valid: std::string const d_name = directory.filename(); if (d_name.size() != FileStorageData::kDirectoryNameLength or not FromHexString(d_name)) { if (FileSystemManager::RemoveDirectory(directory)) { return true; } task.Log(LogLevel::Error, "Failed to remove invalid directory {}", directory.string()); return false; } FileSystemManager::ReadDirEntryFunc callback = [&task, &directory, file_name_size](std::filesystem::path const& file, ObjectType type) -> bool { // Directories are unexpected in storage subdirectories if (IsTreeObject(type)) { task.Log(LogLevel::Error, "There is a directory in a storage subdirectory: {}", (directory / file).string()); return false; } // Check file has a hexadecimal name of length file_name_size: std::string const f_name = file.filename(); if (f_name.size() == file_name_size and FromHexString(f_name)) { return true; } auto const path = directory / file; if (not FileSystemManager::RemoveFile(path)) { task.Log(LogLevel::Error, "Failed to remove invalid entry {}", path.string()); return false; } return true; }; // Read the key storage directory: if (not FileSystemManager::ReadDirectory(directory, callback)) { task.Log(LogLevel::Error, "Failed to read {}", directory.string()); return false; } return true; } template requires(sizeof...(kType) != 0) [[nodiscard]] auto RemoveSpliced(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool { static constexpr bool kLarge = true; auto const directory = task.cas.StorageRoot(kLargeType, kLarge) / key; // Check there are entries to process: if (not FileSystemManager::IsDirectory(directory)) { return true; } // Obtain paths to the corresponding key directories in the object storages. std::array const storage_roots{task.cas.StorageRoot(kType) / key...}; FileSystemManager::ReadDirEntryFunc callback = [&storage_roots, &task, &directory]( std::filesystem::path const& entry_large, ObjectType type) -> bool { // Directories are unexpected in storage subdirectories if (IsTreeObject(type)) { task.Log(LogLevel::Error, "There is a directory in a storage subdirectory: {}", (directory / entry_large).string()); return false; } // Pathes to large entries and spliced results are: // large_storage / entry_large // storage / entry_object // // Large objects are keyed by the hash of their spliced result, so for // splicable objects large_entry and object_entry are the same. // Thus, to check the existence of the spliced result, it is // enough to check the existence of { storage / entry_large }: auto check = [&entry_large](std::filesystem::path const& storage) { std::filesystem::path file_path = storage / entry_large; return not FileSystemManager::IsFile(file_path) or FileSystemManager::RemoveFile(file_path); }; return std::all_of(storage_roots.begin(), storage_roots.end(), check); }; // Read the key storage directory: if (not FileSystemManager::ReadDirectory(directory, callback)) { task.Log(LogLevel::Error, "Failed to read {}", directory.string()); return false; } return true; } template [[nodiscard]] auto SplitLarge(CompactificationTask const& task, std::filesystem::path const& key) noexcept -> bool { auto const path = task.cas.StorageRoot(kType) / key; // Check the entry exists: if (not FileSystemManager::IsFile(path)) { return true; } // Calculate the digest for the entry: auto const digest = ArtifactDigestFactory::HashFileAs( task.cas.GetHashFunction(), path); if (not digest) { task.Log(LogLevel::Error, "Failed to calculate digest for {}", path.string()); return false; } // Split the entry: auto split_result = IsTreeObject(kType) ? task.cas.SplitTree(*digest) : task.cas.SplitBlob(*digest); if (not split_result) { task.Log(LogLevel::Error, "Failed to split {}\nDigest: {}\nMessage: {}", path.string(), digest->hash(), std::move(split_result).error().Message()); return false; } // If the file cannot actually be split (the threshold is too low), the // file must not be deleted. if (split_result->size() < 2) { task.Log(LogLevel::Debug, "{} cannot be compactified. The compactification " "threshold is too low.", digest->hash()); return true; } if (not FileSystemManager::RemoveFile(path)) { task.Log(LogLevel::Error, "Failed to remove {}", path.string()); return false; } return true; } } // namespace just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/compactifier.hpp000066400000000000000000000042351516554100600272310ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFIER_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFIER_HPP #include #include "src/buildtool/storage/local_cas.hpp" class Compactifier final { public: /// \brief Remove invalid entries from the storage. An entry is valid if the /// file and its parent directory have a hexadecimal name of the proper /// size. /// \param cas Storage to be inspected. /// \return True if storage does not contain invalid entries. [[nodiscard]] static auto RemoveInvalid(LocalCAS const& cas) noexcept -> bool; /// \brief Remove spliced entries from the storage. /// \param local_cas Storage to be inspected. /// \return True if object storages do not contain spliced /// entries. [[nodiscard]] static auto RemoveSpliced(LocalCAS const& cas) noexcept -> bool; /// \brief Split and remove from the storage every entry that is larger than /// the compactification threshold. Results of splitting are added to the /// LocalCAS. /// \param local_cas LocalCAS to store results of splitting. /// \param threshold Compactification threshold. /// \return True if the storage doesn't contain splitable /// entries larger than the compactification threshold afterwards. [[nodiscard]] static auto SplitLarge(LocalCAS const& cas, size_t threshold) noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_COMPACTIFIER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/config.hpp000066400000000000000000000217451516554100600260360ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP #include #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/tmp_dir.hpp" struct StorageConfig; struct GenerationConfig final { gsl::not_null const storage_config; std::filesystem::path const cas_f; std::filesystem::path const cas_x; std::filesystem::path const cas_t; std::filesystem::path const cas_large_f; std::filesystem::path const cas_large_t; std::filesystem::path const action_cache; std::filesystem::path const target_cache; }; struct StorageConfig final { class Builder; static inline auto const kDefaultBuildRoot = FileSystemManager::GetUserHome() / ".cache" / "just"; // Build root directory. All the storage dirs are subdirs of build_root. // By default, build_root is set to $HOME/.cache/just. // If the user uses --local-build-root PATH, // then build_root will be set to PATH. std::filesystem::path const build_root = kDefaultBuildRoot; // Number of total storage generations (default: two generations). std::size_t const num_generations = 2; HashFunction const hash_function{HashFunction::Type::GitSHA1}; // Hash of the execution backend description BackendDescription const backend_description; /// \brief Root directory of all storage generations. [[nodiscard]] auto CacheRoot() const noexcept -> std::filesystem::path { return build_root / "protocol-dependent"; } /// \brief Root directory of all repository generations. [[nodiscard]] auto RepositoryRoot() const noexcept -> std::filesystem::path { return build_root / "repositories"; } /// \brief Directory for the given generation of stored repositories [[nodiscard]] auto RepositoryGenerationRoot( std::size_t index) const noexcept -> std::filesystem::path { ExpectsAudit(index < num_generations); auto generation = std::string{"generation-"} + std::to_string(index); return RepositoryRoot() / generation; } /// \brief Directory for the git repository of the given generation [[nodiscard]] auto GitGenerationRoot(std::size_t index) const noexcept -> std::filesystem::path { return RepositoryGenerationRoot(index) / "git"; } /// \brief Directory for the git repository storing various roots [[nodiscard]] auto GitRoot() const noexcept -> std::filesystem::path { return GitGenerationRoot(0); } /// \brief Root directory of specific storage generation [[nodiscard]] auto GenerationCacheRoot(std::size_t index) const noexcept -> std::filesystem::path { ExpectsAudit(index < num_generations); auto generation = std::string{"generation-"} + std::to_string(index); return CacheRoot() / generation; } /// \brief Root directory for all ephemeral directories, i.e., directories /// that can (and should) be removed immediately by garbage collection. [[nodiscard]] auto EphemeralRoot() const noexcept -> std::filesystem::path { return GenerationCacheRoot(0) / "ephemeral"; } /// \brief Root directory for local action executions; individual actions /// create a working directory below this root. [[nodiscard]] auto ExecutionRoot() const noexcept -> std::filesystem::path { return EphemeralRoot() / "exec_root"; } /// \brief Create a tmp directory with controlled lifetime for specific /// operations (archive, zip, file, distdir checkouts; fetch; update). [[nodiscard]] auto CreateTypedTmpDir(std::string const& type) const noexcept -> TmpDir::Ptr { // try to create parent dir auto parent_path = EphemeralRoot() / "tmp-workspaces" / type; return TmpDir::Create(parent_path); } [[nodiscard]] auto CreateGenerationConfig( std::size_t generation) const noexcept -> GenerationConfig { bool const native = ProtocolTraits::IsNative(hash_function.GetType()); auto const cache_root = GenerationCacheRoot(generation); auto const cache_dir = UpdatePathForCompatibility(cache_root, native); return GenerationConfig{ .storage_config = this, .cas_f = cache_dir / "casf", .cas_x = cache_dir / "casx", .cas_t = cache_dir / (native ? "cast" : "casf"), .cas_large_f = cache_dir / "cas-large-f", .cas_large_t = cache_dir / (native ? "cas-large-t" : "cas-large-f"), .action_cache = cache_dir / "ac", .target_cache = cache_dir / "tc"}; }; private: // different folder for different caching protocol [[nodiscard]] static auto UpdatePathForCompatibility( std::filesystem::path const& dir, bool is_native) -> std::filesystem::path { return dir / (is_native ? "git-sha1" : "compatible-sha256"); }; }; class StorageConfig::Builder final { public: explicit Builder() = default; /// \brief Create a configurable builder from an existing config. /// Useful, for example, to make a copy of an existing config and change /// the hash type. [[nodiscard]] static auto Rebuild(StorageConfig const& config) noexcept -> Builder { return Builder{} .SetBuildRoot(config.build_root) .SetNumGenerations(config.num_generations) .SetHashType(config.hash_function.GetType()) .SetBackendDescription(config.backend_description); } auto SetBuildRoot(std::filesystem::path value) noexcept -> Builder& { build_root_ = std::move(value); return *this; } /// \brief Specifies the number of storage generations. auto SetNumGenerations(std::size_t value) noexcept -> Builder& { num_generations_ = value; return *this; } /// \brief Specify the type of the hash function auto SetHashType(HashFunction::Type value) noexcept -> Builder& { hash_type_ = value; return *this; } auto SetBackendDescription(BackendDescription value) noexcept -> Builder& { backend_description_ = std::move(value); return *this; } [[nodiscard]] auto Build() const noexcept -> expected { // To not duplicate default arguments of StorageConfig in builder, // create a default config and copy default arguments from there. StorageConfig const default_config; auto build_root = default_config.build_root; if (build_root_.has_value()) { build_root = *build_root_; if (FileSystemManager::IsRelativePath(build_root)) { return unexpected(fmt::format( "Build root must be absolute path but got '{}'.", build_root.string())); } } auto num_generations = default_config.num_generations; if (num_generations_.has_value()) { num_generations = *num_generations_; if (num_generations == 0) { return unexpected(std::string{ "The number of generations must be greater than 0."}); } } auto const hash_function = hash_type_.has_value() ? HashFunction{*hash_type_} : default_config.hash_function; // Hash the execution backend description auto backend_description = default_config.backend_description; if (backend_description_) { backend_description = *backend_description_; } return StorageConfig{ .build_root = std::move(build_root), .num_generations = num_generations, .hash_function = hash_function, .backend_description = std::move(backend_description)}; } private: std::optional build_root_; std::optional num_generations_; std::optional hash_type_; std::optional backend_description_; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/file_chunker.cpp000066400000000000000000000072551516554100600272220ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/file_chunker.hpp" #include #include #include "gsl/gsl" namespace { // Mask values taken from algorithm 2 of the paper // https://ieeexplore.ieee.org/document/9055082. constexpr std::uint64_t kMaskS{0x4444d9f003530000ULL}; // 19 '1' bits constexpr std::uint64_t kMaskL{0x4444d90003530000ULL}; // 15 '1' bits // Predefined array of 256 random 64-bit integers, needs to be initialized. constexpr std::uint32_t kRandomTableSize{256}; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::array gear_table{}; } // namespace auto FileChunker::Initialize(std::uint32_t seed) noexcept -> void { std::mt19937_64 gen64(seed); for (auto& item : gear_table) { item = gen64(); } } auto FileChunker::IsOpen() const noexcept -> bool { return stream_.is_open(); } auto FileChunker::Finished() const noexcept -> bool { return stream_.eof() and pos_ == size_; } auto FileChunker::NextChunk() noexcept -> std::optional { // Handle failed past read attempts from the stream. if (not stream_.good() and not stream_.eof()) { return std::nullopt; } // Ensure that at least max_chunk_size bytes are in the buffer, except if // end-of-file is reached. auto remaining = size_ - pos_; if (remaining < max_chunk_size_ and not stream_.eof()) { // Move the remaining bytes of the buffer to the front. buffer_.copy(buffer_.data(), remaining, pos_); auto ssize = static_cast(buffer_.size() - remaining); // Fill the buffer with stream content. stream_.read(&buffer_[remaining], ssize); if (not stream_.good() and not stream_.eof()) { return std::nullopt; } size_ = static_cast(stream_.gcount()) + remaining; pos_ = 0; } // Handle finished chunking. if (pos_ == size_) { return std::nullopt; } auto off = NextChunkBoundary(); auto chunk = buffer_.substr(pos_, off); pos_ += off; return chunk; } // Implementation of the FastCDC data deduplication algorithm described in // algorithm 2 of the paper https://ieeexplore.ieee.org/document/9055082. auto FileChunker::NextChunkBoundary() noexcept -> std::size_t { auto n = size_ - pos_; auto fp = 0ULL; auto i = min_chunk_size_; auto normal_size = average_chunk_size_; if (n <= min_chunk_size_) { return n; } if (n >= max_chunk_size_) { n = max_chunk_size_; } else if (n <= normal_size) { normal_size = n; } for (; i < normal_size; i++) { fp = (fp << 1U) + gsl::at(gear_table, static_cast(buffer_[pos_ + i])); if ((fp & kMaskS) == 0) { return i; // if the masked bits are all '0' } } for (; i < n; i++) { fp = (fp << 1U) + gsl::at(gear_table, static_cast(buffer_[pos_ + i])); if ((fp & kMaskL) == 0) { return i; // if the masked bits are all '0' } } return i; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/file_chunker.hpp000066400000000000000000000104761516554100600272260ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_FILE_CHUNKER_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_FILE_CHUNKER_HPP #include #include #include #include #include #include /// @brief This class provides content-defined chunking for a file stream. It /// allows to split a file stream into variable-sized chunks based on its data /// content. In contrast to fixed-sized chunking, which splits a data stream /// into chunks of fixed size, it is not prone to the data-shifting problem. In /// order to assemble the resulting file, the delivered chunks have to be /// concatenated in order. /// /// A read buffer is used to progressively process the file content instead of /// reading the entire file content in memory. class FileChunker { static constexpr std::uint32_t kAverageChunkSize{1024 * 128}; // 128 KB static constexpr std::uint32_t kDefaultSeed{0}; public: /// @brief Create an instance of the file chunker for a given file. /// @param path The path to the file to be splitted. /// @param average_chunk_size Targeted average chunk size in bytes /// (default: 8 KB). explicit FileChunker(std::filesystem::path const& path) // According to section 4.1 of the paper // https://ieeexplore.ieee.org/document/9055082, maximum and minimum // chunk sizes are configured to the 8x and the 1/4x of the average // chunk size. : min_chunk_size_(kAverageChunkSize >> 2U), average_chunk_size_(kAverageChunkSize), max_chunk_size_(kAverageChunkSize << 3U), stream_{path, std::ios::in | std::ios::binary} { // The buffer size needs to be at least max_chunk_size_ large, otherwise // max_chunk_size_ is not fully exhausted and the buffer size determines // the maximum chunk size. buffer_.resize(max_chunk_size_ << 1U); } FileChunker() noexcept = delete; ~FileChunker() noexcept = default; FileChunker(FileChunker const& other) noexcept = delete; FileChunker(FileChunker&& other) noexcept = delete; auto operator=(FileChunker const& other) noexcept = delete; auto operator=(FileChunker&& other) noexcept = delete; /// @brief Check if the underlying file is open. /// @return True if the file was opened successfully, false otherwise. [[nodiscard]] auto IsOpen() const noexcept -> bool; /// @brief Check if chunking of the file stream was done successfully. /// @return True if chunking was successful, false otherwise. [[nodiscard]] auto Finished() const noexcept -> bool; /// @brief Fetch the next chunk from the file stream. /// @return The next chunk of the file stream. [[nodiscard]] auto NextChunk() noexcept -> std::optional; /// @brief Initialize random number table used by the chunking algorithm. /// @param seed Some random seed. static auto Initialize(std::uint32_t seed = kDefaultSeed) noexcept -> void; private: // Different chunk size parameters, defined in number of bytes. const std::uint32_t min_chunk_size_{}; const std::uint32_t average_chunk_size_{}; const std::uint32_t max_chunk_size_{}; std::ifstream stream_; // File stream to be splitted. std::string buffer_; // Buffer for the file content. std::size_t size_{0}; // Current size of the buffer. std::size_t pos_{0}; // Current read position within the buffer. /// @brief Find the next chunk boundary from the current read position /// within the buffer. /// @return The position of the next chunk boundary. [[nodiscard]] auto NextChunkBoundary() noexcept -> std::size_t; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_FILE_CHUNKER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/fs_utils.cpp000066400000000000000000000152761516554100600264160ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/fs_utils.hpp" #include #include #include //std::ignore #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace StorageUtils { auto GetGitRoot(StorageConfig const& storage_config, LocalPathsPtr const& just_mr_paths, std::string const& repo_url) noexcept -> std::filesystem::path { if (just_mr_paths->git_checkout_locations.contains(repo_url)) { if (just_mr_paths->git_checkout_locations[repo_url].is_string()) { return std::filesystem::absolute(ToNormalPath(std::filesystem::path{ just_mr_paths->git_checkout_locations[repo_url] .get()})); } Logger::Log( LogLevel::Warning, "Retrieving Git checkout location: key {} has non-string value {}", nlohmann::json(repo_url).dump(), just_mr_paths->git_checkout_locations[repo_url].dump()); } auto repo_url_as_path = std::filesystem::absolute( ToNormalPath(std::filesystem::path(repo_url))); if (not repo_url_as_path.empty() and FileSystemManager::IsAbsolutePath(repo_url_as_path) and FileSystemManager::IsDirectory(repo_url_as_path)) { return repo_url_as_path; } return storage_config.GitRoot(); } auto GetCommitTreeIDFile(StorageConfig const& storage_config, std::string const& commit, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.RepositoryGenerationRoot(generation) / "commit-tree-map" / commit; } auto GetArchiveTreeIDFile(StorageConfig const& storage_config, std::string const& repo_type, std::string const& content, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.RepositoryGenerationRoot(generation) / "tree-map" / repo_type / content; } auto GetForeignFileTreeIDFile(StorageConfig const& storage_config, std::string const& content, std::string const& name, bool executable, std::size_t generation) noexcept -> std::filesystem::path { return GetDistdirTreeIDFile( storage_config, storage_config.hash_function .HashBlobData( nlohmann::json(std::unordered_map>{ {name, {content, executable}}}) .dump()) .HexString(), generation); } auto GetDistdirTreeIDFile(StorageConfig const& storage_config, std::string const& content, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.RepositoryGenerationRoot(generation) / "distfiles-tree-map" / content; } auto GetResolvedTreeIDFile(StorageConfig const& storage_config, std::string const& tree_hash, PragmaSpecial const& pragma_special, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.RepositoryGenerationRoot(generation) / "special-tree-map" / kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; } auto GetRehashIDFile(StorageConfig const& storage_config, HashFunction::Type target_hash_type, std::string const& hash, bool from_git, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.GenerationCacheRoot(generation) / fmt::format("to-{}", ToString(target_hash_type)) / (from_git ? "from-git" : "from-cas") / hash; } auto GetValidTreesMarkerFile(StorageConfig const& storage_config, std::string const& tree_hash, std::size_t generation) noexcept -> std::filesystem::path { return storage_config.GenerationCacheRoot(generation) / "validated-git-trees" / tree_hash; } auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, std::string const& tree_id) noexcept -> bool { // needs to be done safely, so use the rename trick auto tmp_dir = TmpDir::Create(tree_id_file.parent_path()); if (not tmp_dir) { Logger::Log(LogLevel::Error, "could not create tmp dir for writing tree id file {}", tree_id_file.string()); return false; } std::filesystem::path tmp_file{tmp_dir->GetPath() / "tmp_file"}; if (not FileSystemManager::WriteFile(tree_id, tmp_file)) { Logger::Log(LogLevel::Error, "could not create tmp tree id file"); return false; } return FileSystemManager::Rename(tmp_file.string(), tree_id_file); } auto AddToCAS(Storage const& storage, std::string const& data) noexcept -> std::optional { // get file CAS instance auto const& cas = storage.CAS(); // write to cas if (auto digest = cas.StoreBlob(data)) { return cas.BlobPath(*digest, /*is_executable=*/false); } return std::nullopt; } void AddDistfileToCAS(Storage const& storage, std::filesystem::path const& distfile, LocalPathsPtr const& just_mr_paths) noexcept { auto const& cas = storage.CAS(); for (auto const& dirpath : just_mr_paths->distdirs) { auto candidate = dirpath / distfile; if (FileSystemManager::Exists(candidate)) { // try to add to CAS std::ignore = cas.StoreBlob(candidate, /*is_executable=*/false); } } } } // namespace StorageUtils just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/fs_utils.hpp000066400000000000000000000124131516554100600264110ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP #include #include #include #include #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" /* Utilities related to CAS and paths therein */ namespace StorageUtils { /// \brief Get location of Git repository. Defaults to the Git cache root when /// no better location is found. [[nodiscard]] auto GetGitRoot(StorageConfig const& storage_config, LocalPathsPtr const& just_mr_paths, std::string const& repo_url) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id associated with /// a given commit. [[nodiscard]] auto GetCommitTreeIDFile(StorageConfig const& storage_config, std::string const& commit, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of an archive /// content. [[nodiscard]] auto GetArchiveTreeIDFile(StorageConfig const& storage_config, std::string const& repo_type, std::string const& content, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of an archive /// content. [[nodiscard]] auto GetForeignFileTreeIDFile(StorageConfig const& storage_config, std::string const& content, std::string const& name, bool executable, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of a distdir list /// content. [[nodiscard]] auto GetDistdirTreeIDFile(StorageConfig const& storage_config, std::string const& content, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing a resolved tree hash. [[nodiscard]] auto GetResolvedTreeIDFile(StorageConfig const& storage_config, std::string const& tree_hash, PragmaSpecial const& pragma_special, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the corresponding artifact hashed by /// a different hash function. /// \param storage_config Storage under which the file is to be found. /// \param target_hash_type Hash type to identify mapping target. /// \param hash Hash to identify mapping source. /// \param from_git Flag to distinguish further mapping source (CAS / GitCAS) /// \param generation Further specificity in location of the file. [[nodiscard]] auto GetRehashIDFile(StorageConfig const& storage_config, HashFunction::Type target_hash_type, std::string const& hash, bool from_git, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Get the path to the file marking a known valid Git tree. [[nodiscard]] auto GetValidTreesMarkerFile(StorageConfig const& storage_config, std::string const& tree_hash, std::size_t generation = 0) noexcept -> std::filesystem::path; /// \brief Write a tree id to file. The parent folder of the file must exist! [[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, std::string const& tree_id) noexcept -> bool; /// \brief Add data to file CAS. /// Returns the path to the file added to CAS, or nullopt if not added. [[nodiscard]] auto AddToCAS(Storage const& storage, std::string const& data) noexcept -> std::optional; /// \brief Try to add distfile to CAS. void AddDistfileToCAS(Storage const& storage, std::filesystem::path const& distfile, LocalPathsPtr const& just_mr_paths) noexcept; } // namespace StorageUtils #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/garbage_collector.cpp000066400000000000000000000246501516554100600302200ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/storage/garbage_collector.hpp" #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/compactifier.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" namespace { auto RemoveDirs(const std::vector& directories) -> bool { bool success = true; for (auto const& d : directories) { if (FileSystemManager::IsDirectory(d)) { if (not FileSystemManager::RemoveDirectory(d)) { Logger::Log(LogLevel::Warning, "Failed to remove directory {}", d.string()); success = false; // We failed to remove all, but still try the // others to clean up as much as possible } } } return success; } } // namespace auto GarbageCollector::SharedLock(StorageConfig const& storage_config) noexcept -> std::optional { return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/true); } auto GarbageCollector::ExclusiveLock( StorageConfig const& storage_config) noexcept -> std::optional { return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/false); } auto GarbageCollector::LockFilePath( StorageConfig const& storage_config) noexcept -> std::filesystem::path { return storage_config.CacheRoot() / "gc.lock"; } auto GarbageCollector::TriggerGarbageCollection( StorageConfig const& storage_config, bool no_rotation, bool gc_all) noexcept -> bool { if (no_rotation and gc_all) { Logger::Log(LogLevel::Error, "no_rotation and gc_all cannot be set together"); return false; } std::string const remove_me = "remove-me"; auto pid = CreateProcessUniqueId(); if (not pid) { return false; } auto remove_me_prefix = remove_me + *pid + std::string{"-"}; std::vector to_remove{}; // With a shared lock, we can remove all directories with the given prefix, // as we own the process id. { auto lock = SharedLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to get a shared lock the local build root"); return false; } for (auto const& entry : std::filesystem::directory_iterator(storage_config.CacheRoot())) { if (entry.path().filename().string().starts_with( remove_me_prefix)) { to_remove.emplace_back(entry.path()); } } if (not RemoveDirs(to_remove)) { Logger::Log(LogLevel::Error, "Failed to clean up left-over directories under my " "pid. Will not continue"); return false; } } to_remove.clear(); int remove_me_counter{}; // after releasing the shared lock, wait to get an exclusive lock for doing // the critical renaming { auto lock = ExclusiveLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to exclusively lock the local build root"); return false; } // First, while he have not yet created any to-remove directories, grab // all existing remove-me directories; they're left overs, as the clean // up of owned directories is done with a shared lock. std::vector left_over{}; for (auto const& entry : std::filesystem::directory_iterator(storage_config.CacheRoot())) { if (entry.path().filename().string().starts_with(remove_me)) { left_over.emplace_back(entry.path()); } } for (auto const& d : left_over) { auto new_name = storage_config.CacheRoot() / fmt::format("{}{}", remove_me_prefix, remove_me_counter++); if (FileSystemManager::Rename(d, new_name)) { to_remove.emplace_back(new_name); } else { Logger::Log(LogLevel::Warning, "Failed to rename {} to {}", d.string(), new_name.string()); } } // Now that the have to exclusive lock, try to move out ephemeral data; // as it is still under the generation regime, it is not a huge problem // if that fails. auto ephemeral = storage_config.EphemeralRoot(); if (FileSystemManager::IsDirectory(ephemeral)) { auto remove_me_dir = storage_config.CacheRoot() / fmt::format("{}{}", remove_me_prefix, remove_me_counter++); if (FileSystemManager::Rename(ephemeral, remove_me_dir)) { to_remove.emplace_back(remove_me_dir); } else { Logger::Log(LogLevel::Warning, "Failed to rename {} to {}.", ephemeral.string(), remove_me_dir.string()); } } // Compactification must take place before rotating generations. // Otherwise, an interruption of the process during compactification // would lead to an invalid old generation. if (not gc_all and not GarbageCollector::Compactify( storage_config, MessageLimits::kMaxGrpcLength)) { Logger::Log(LogLevel::Error, "Failed to compactify the youngest generation."); return false; } if (gc_all) { // Rename all cache generations starting from the oldest generation. // If the process gets interrupted, the youngest cache stays // available. for (int i = static_cast(storage_config.num_generations) - 1; i >= 0; --i) { auto remove_me_dir = storage_config.CacheRoot() / fmt::format("{}{}", remove_me_prefix, remove_me_counter++); to_remove.emplace_back(remove_me_dir); auto cache_root = storage_config.GenerationCacheRoot(i); if (FileSystemManager::IsDirectory(cache_root) and not FileSystemManager::Rename(cache_root, remove_me_dir)) { Logger::Log(LogLevel::Error, "Failed to rename {} to {}", cache_root.string(), remove_me_dir.string()); return false; } } } // Rotate generations unless told not to do so else if (not no_rotation) { auto remove_me_dir = storage_config.CacheRoot() / fmt::format("{}{}", remove_me_prefix, remove_me_counter++); to_remove.emplace_back(remove_me_dir); for (std::size_t i = storage_config.num_generations; i > 0; --i) { auto cache_root = storage_config.GenerationCacheRoot(i - 1); if (FileSystemManager::IsDirectory(cache_root)) { auto new_cache_root = (i == storage_config.num_generations) ? remove_me_dir : storage_config.GenerationCacheRoot(i); if (not FileSystemManager::Rename(cache_root, new_cache_root)) { Logger::Log(LogLevel::Error, "Failed to rename {} to {}.", cache_root.string(), new_cache_root.string()); return false; } } } } } // After releasing the exclusive lock, get a shared lock and remove what we // have to remove bool success{}; { auto lock = SharedLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to get a shared lock the local build root"); return false; } success = RemoveDirs(to_remove); } return success; } auto GarbageCollector::Compactify(StorageConfig const& storage_config, size_t threshold) noexcept -> bool { Logger::Log(LogLevel::Performance, "Compactification has been started"); // Compactification must be done for both native and compatible storages. static constexpr std::array kHashes = {HashFunction::Type::GitSHA1, HashFunction::Type::PlainSHA256}; return std::all_of( kHashes.begin(), kHashes.end(), [threshold, &storage_config](HashFunction::Type hash_type) { auto const config = StorageConfig::Builder::Rebuild(storage_config) .SetHashType(hash_type) .Build(); if (not config) { return false; } auto const storage = ::Generation::Create(&*config); return Compactifier::RemoveInvalid(storage.CAS()) and Compactifier::RemoveSpliced(storage.CAS()) and Compactifier::SplitLarge(storage.CAS(), threshold); }); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/garbage_collector.hpp000066400000000000000000000052571516554100600302270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP #include #include #include #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/file_locking.hpp" /// \brief Global garbage collector implementation. /// Responsible for deleting oldest generation. class GarbageCollector { public: /// \brief Trigger garbage collection /// \param storage_config Storage to collect garbage in /// \param no_rotation Skip rotation of generation and perform only steps /// that can be done without losing existing cache. Incompatible with /// gc_all. /// \param gc_all Remove all cache generations at once. Incompatible with /// no_rotation. /// \return true on success. [[nodiscard]] auto static TriggerGarbageCollection( StorageConfig const& storage_config, bool no_rotation = false, bool gc_all = false) noexcept -> bool; /// \brief Acquire shared lock to prevent garbage collection from running. /// \param storage_config Storage to be locked. /// \returns The acquired lock file on success or nullopt otherwise. [[nodiscard]] auto static SharedLock( StorageConfig const& storage_config) noexcept -> std::optional; private: [[nodiscard]] auto static ExclusiveLock( StorageConfig const& storage_config) noexcept -> std::optional; [[nodiscard]] auto static LockFilePath( StorageConfig const& storage_config) noexcept -> std::filesystem::path; /// \brief Remove spliced objects from the youngest generation and split /// objects that are larger than the threshold. /// \param threshold Compactification threshold. /// \return True if the youngest generation does not contain splicable /// objects afterwards. [[nodiscard]] auto static Compactify(StorageConfig const& storage_config, size_t threshold) noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/large_object_cas.hpp000066400000000000000000000170651516554100600300370ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/uplinker.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" template class LocalCAS; enum class LargeObjectErrorCode : std::uint8_t { /// \brief An internal error occurred. Internal = 0, /// \brief The digest is not in the CAS. FileNotFound, /// \brief The result is different from what was expected. InvalidResult, /// \brief Some parts of the tree are not in the storage. InvalidTree }; /// \brief Describes an error that occurred during split-splice. class LargeObjectError final { public: LargeObjectError(LargeObjectErrorCode code, std::string message) : code_(code), message_(std::move(message)) {} /// \brief Obtain the error code. [[nodiscard]] auto Code() const noexcept -> LargeObjectErrorCode { return code_; } /// \brief Obtain the error message. [[nodiscard]] auto Message() && noexcept -> std::string { return std::move(message_); } private: LargeObjectErrorCode const code_; std::string message_; }; /// \brief Stores auxiliary information for reconstructing large objects. /// The entries are keyed by the hash of the spliced result and the value of an /// entry is the concatenation of the hashes of chunks the large object is /// composed of. template class LargeObjectCAS final { public: explicit LargeObjectCAS( gsl::not_null const*> const& local_cas, GenerationConfig const& config, gsl::not_null const*> const& uplinker) noexcept : local_cas_(*local_cas), storage_config_{*config.storage_config}, uplinker_{*uplinker}, file_store_(IsTreeObject(kType) ? config.cas_large_t : config.cas_large_f) {} LargeObjectCAS(LargeObjectCAS const&) = delete; LargeObjectCAS(LargeObjectCAS&&) = delete; auto operator=(LargeObjectCAS const&) -> LargeObjectCAS& = delete; auto operator=(LargeObjectCAS&&) -> LargeObjectCAS& = delete; ~LargeObjectCAS() noexcept = default; /// \brief Obtain path to the storage root. [[nodiscard]] auto StorageRoot() const noexcept -> std::filesystem::path const& { return file_store_.StorageRoot(); } /// \brief Get the path to a large entry in the storage. /// \param digest The digest of a large object. /// \returns Path to the large entry if in the storage. [[nodiscard]] auto GetEntryPath(ArtifactDigest const& digest) const noexcept -> std::optional; /// \brief Split an object from the main CAS into chunks. If the object had /// been split before, it would not get split again. /// \param digest The digest of the object to be split. /// \return A set of chunks the resulting object is composed of /// or an error on failure. [[nodiscard]] auto Split(ArtifactDigest const& digest) const noexcept -> expected, LargeObjectError>; /// \brief Splice an object based on the reconstruction rules from the /// storage. This method doesn't check whether the result of splicing is /// already in the CAS. /// \param digest The digest of the object to be spliced. /// \return A temporary directory that contains a single file /// "result" on success or an error on failure. [[nodiscard]] auto TrySplice(ArtifactDigest const& digest) const noexcept -> expected; /// \brief Splice an object from parts. This method doesn't check whether /// the result of splicing is already in the CAS. /// \param digest The digest of the resulting object. /// \param parts Parts to be concatenated. /// \return A temporary directory that contains a single file /// "result" on success or an error on failure. [[nodiscard]] auto Splice(ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected; /// \brief Uplink large entry from this generation to latest LocalCAS /// generation. For the large entry it's parts get promoted first and then /// the entry itself. This function is only available for instances that are /// used as local GC generations (i.e., disabled global uplink). /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest LocalCAS generation. /// \param latest_large The latest LargeObjectCAS /// \param digest The digest of the large entry to uplink. /// \returns True if the large entry was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplink( LocalCAS const& latest, LargeObjectCAS const& latest_large, ArtifactDigest const& digest) const noexcept -> bool; private: // By default, overwrite existing entries. Unless this is a generation // (disabled global uplink), then we never want to overwrite any entries. static constexpr auto kStoreMode = kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; LocalCAS const& local_cas_; StorageConfig const& storage_config_; Uplinker const& uplinker_; FileStorage file_store_; /// \brief Obtain the information for reconstructing a large object. /// \param digest The digest of a large object. /// \returns Parts the large object is composed of, if present in /// the storage. [[nodiscard]] auto ReadEntry(ArtifactDigest const& digest) const noexcept -> std::optional>; /// \brief Create a new entry description and add it to the storage. /// \param digest The digest of the result. /// \param parts Parts the resulting object is composed of. /// \returns True if the entry exists afterwards. [[nodiscard]] auto WriteEntry( ArtifactDigest const& digest, std::vector const& parts) const noexcept -> bool; }; // NOLINTNEXTLINE(misc-header-include-cycle) #include "src/buildtool/storage/large_object_cas.tpp" // IWYU pragma: export #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/large_object_cas.tpp000066400000000000000000000237441516554100600300540ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP // IWYU pragma: private, include "src/buildtool/storage/large_object_cas.hpp" #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/file_chunker.hpp" #include "src/buildtool/storage/large_object_cas.hpp" #include "src/buildtool/storage/local_cas.hpp" namespace { inline constexpr std::size_t kHashIndex = 0; inline constexpr std::size_t kSizeIndex = 1; } // namespace template auto LargeObjectCAS::GetEntryPath( ArtifactDigest const& digest) const noexcept -> std::optional { std::filesystem::path file_path = file_store_.GetPath(digest.hash()); if (FileSystemManager::IsFile(file_path)) { return file_path; } if constexpr (kDoGlobalUplink) { // To promote parts of the tree properly, regular uplinking logic for // trees is used: auto const hash_type = storage_config_.hash_function.GetType(); bool const uplinked = IsTreeObject(kType) and not ProtocolTraits::IsTreeAllowed(hash_type) ? uplinker_.UplinkTree(digest) : uplinker_.UplinkLargeBlob(digest); if (uplinked and FileSystemManager::IsFile(file_path)) { return file_path; } } return std::nullopt; } template auto LargeObjectCAS::ReadEntry( ArtifactDigest const& digest) const noexcept -> std::optional> { auto const file_path = GetEntryPath(digest); if (not file_path) { return std::nullopt; } std::vector parts; try { std::ifstream stream(*file_path); nlohmann::json j = nlohmann::json::parse(stream); parts.reserve(j.size()); auto const hash_type = local_cas_.GetHashFunction().GetType(); for (auto const& j_part : j) { auto digest = ArtifactDigestFactory::Create( hash_type, j_part.at(kHashIndex).template get(), j_part.at(kSizeIndex).template get(), /*is_tree=*/false); if (not digest) { return std::nullopt; } parts.emplace_back(*std::move(digest)); } } catch (...) { return std::nullopt; } return parts; } template auto LargeObjectCAS::WriteEntry( ArtifactDigest const& digest, std::vector const& parts) const noexcept -> bool { if (GetEntryPath(digest)) { return true; } // The large entry cannot refer itself or be empty. // Otherwise, the digest in the main CAS would be removed during GC. // It would bring the LargeObjectCAS to an invalid state: the // large entry exists, but the parts do not. if (parts.size() < 2) { return false; } nlohmann::json j; try { for (auto const& part : parts) { auto& j_part = j.emplace_back(); j_part[kHashIndex] = part.hash(); j_part[kSizeIndex] = part.size(); } } catch (...) { return false; } return file_store_.AddFromBytes(digest.hash(), j.dump()); } template auto LargeObjectCAS::Split(ArtifactDigest const& digest) const noexcept -> expected, LargeObjectError> { if (auto large_entry = ReadEntry(digest)) { return std::move(*large_entry); } // Get path to the file: std::optional file_path; if constexpr (IsTreeObject(kType)) { file_path = local_cas_.TreePath(digest); } else { // Avoid synchronization between file/executable storages: static constexpr bool kIsExec = IsExecutableObject(kType); file_path = local_cas_.BlobPathNoSync(digest, kIsExec); file_path = file_path ? file_path : local_cas_.BlobPathNoSync(digest, not kIsExec); } if (not file_path) { return unexpected{ LargeObjectError{LargeObjectErrorCode::FileNotFound, fmt::format("could not find {}", digest.hash())}}; } // Split file into chunks: FileChunker chunker{*file_path}; if (not chunker.IsOpen()) { return unexpected{ LargeObjectError{LargeObjectErrorCode::Internal, fmt::format("could not split {}", digest.hash())}}; } std::vector parts; try { while (auto chunk = chunker.NextChunk()) { auto part = local_cas_.StoreBlob(*chunk, /*is_executable=*/false); if (not part) { return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, "could not store a part."}}; } parts.emplace_back(*std::move(part)); } } catch (...) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, "an unknown error occurred."}}; } if (not chunker.Finished()) { return unexpected{ LargeObjectError{LargeObjectErrorCode::Internal, fmt::format("could not split {}", digest.hash())}}; } std::ignore = WriteEntry(digest, parts); return parts; } template auto LargeObjectCAS::TrySplice( ArtifactDigest const& digest) const noexcept -> expected { auto parts = ReadEntry(digest); if (not parts) { return unexpected{LargeObjectError{ LargeObjectErrorCode::FileNotFound, fmt::format("could not find large entry for {}", digest.hash())}}; } return Splice(digest, *parts); } template auto LargeObjectCAS::Splice( ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected { // Create temporary space for splicing: TmpFile::Ptr large_object = TmpDir::CreateFile( storage_config_.CreateTypedTmpDir("splice"), /*file_name=*/"result"); if (large_object == nullptr) { return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not create a temporary space for {}", digest.hash())}}; } // Splice the object from parts try { std::ofstream stream(large_object->GetPath()); for (auto const& part : parts) { auto part_path = local_cas_.BlobPath(part, /*is_executable=*/false); if (not part_path) { return unexpected{LargeObjectError{ LargeObjectErrorCode::FileNotFound, fmt::format("could not find the part {}", part.hash())}}; } auto part_content = FileSystemManager::ReadFile(*part_path); if (not part_content) { return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not read the part content {}", part.hash())}}; } if (stream.good()) { stream << *part_content; } else { return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not splice {}", digest.hash())}}; } } stream.close(); } catch (...) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, "an unknown error occurred"}}; } return large_object; } template template requires(kIsLocalGeneration) auto LargeObjectCAS::LocalUplink( LocalCAS const& latest, LargeObjectCAS const& latest_large, ArtifactDigest const& digest) const noexcept -> bool { // Check the large entry in the youngest generation: if (latest_large.GetEntryPath(digest)) { return true; } // Check the large entry in the current generation: auto parts = ReadEntry(digest); if (not parts) { // No large entry or the object is not large return true; } // Promoting the parts of the large entry: for (auto const& part : *parts) { static constexpr bool kIsExecutable = false; static constexpr bool kSkipSync = true; if (not local_cas_.LocalUplinkBlob( latest, part, kIsExecutable, kSkipSync)) { return false; } } auto path = GetEntryPath(digest); if (not path) { return false; } return latest_large.file_store_.AddFromFile( digest.hash(), *path, /*is_owner=*/true); } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/local_ac.hpp000066400000000000000000000143511516554100600263210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/uplinker.hpp" #include "src/utils/cpp/expected.hpp" #ifdef BOOTSTRAP_BUILD_TOOL // forward declarations needed by the bootstrap process only namespace build::bazel::remote::execution::v2 { class ActionResult; } // namespace build::bazel::remote::execution::v2 #endif // BOOTSTRAP_BUILD_TOOL /// \brief The action cache for storing action results. /// Supports global uplinking across all generations. The uplink is /// automatically performed for every entry that is read and already exists in /// an older generation. /// \tparam kDoGlobalUplink Enable global uplinking. template class LocalAC { public: /// Local AC generation used by GC without global uplink. using LocalGenerationAC = LocalAC; explicit LocalAC(gsl::not_null const*> const& cas, GenerationConfig const& config, gsl::not_null const*> const& uplinker) noexcept : cas_{*cas}, file_store_{config.action_cache}, uplinker_{*uplinker} {} LocalAC(LocalAC const&) = default; LocalAC(LocalAC&&) noexcept = default; auto operator=(LocalAC const&) -> LocalAC& = default; auto operator=(LocalAC&&) noexcept -> LocalAC& = default; ~LocalAC() noexcept = default; /// \brief Store action result. /// \param action_id The id of the action that produced the result. /// \param result The action result to store. /// \returns true on success. [[nodiscard]] auto StoreResult( ArtifactDigest const& action_id, bazel_re::ActionResult const& result) const noexcept -> bool; /// \brief Read cached action result. /// \param action_id The id of the action the result was produced by. /// \returns The action result if found or nullopt otherwise. [[nodiscard]] auto CachedResult(ArtifactDigest const& action_id) const noexcept -> std::optional; /// \brief Uplink entry from this generation to latest LocalAC generation. /// This function is only available for instances that are used as local GC /// generations (i.e., disabled global uplink). /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest LocalAC generation. /// \param action_id The id of the action used as entry key. /// \returns True if entry was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( LocalGenerationAC const& latest, ArtifactDigest const& action_id) const noexcept -> bool; private: // The action cache stores the results of failed actions. For those to be // overwritable by subsequent runs we need to choose the store mode "last // wins" for the underlying file storage. Unless this is a generation // (disabled global uplink), then we never want to overwrite any entries. static constexpr auto kStoreMode = kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; std::shared_ptr logger_{std::make_shared("LocalAC")}; LocalCAS const& cas_; FileStorage file_store_; Uplinker const& uplinker_; /// \brief Add an entry to the ActionCache. /// \param action_id The id of the action that produced the result. /// \param cas_key The key pointing at an ActionResult in the LocalCAS. /// \return True if an entry was successfully added to the storage. [[nodiscard]] auto WriteActionKey( ArtifactDigest const& action_id, ArtifactDigest const& cas_key) const noexcept -> bool; /// \brief Get the key pointing at an ActionResult in the LocalCAS. /// \param action_id The id of the action that produced the result. /// \return The key of an Action pointing at an ActionResult in the LocalCAS /// on success or an error message on failure. [[nodiscard]] auto ReadActionKey(ArtifactDigest const& action_id) const noexcept -> expected; /// \brief Add an action to the LocalCAS. /// \param action The action result to store. /// \return The key pointing at an ActionResult present in the LocalCAS on /// success or std::nullopt on failure. [[nodiscard]] auto WriteAction(bazel_re::ActionResult const& action) const noexcept -> std::optional; /// \brief Get the action specified by a key from the LocalCAS. /// \param cas_key The key pointing at an ActionResult present in the /// LocalCAS. /// \return The ActionResult corresponding to a cas_key on success /// or std::nullopt on failure. [[nodiscard]] auto ReadAction(ArtifactDigest const& cas_key) const noexcept -> std::optional; }; #ifndef BOOTSTRAP_BUILD_TOOL // NOLINTNEXTLINE(misc-header-include-cycle) #include "src/buildtool/storage/local_ac.tpp" // IWYU pragma: export #endif #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/local_ac.tpp000066400000000000000000000157631516554100600263450ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP // IWYU pragma: private, include "src/buildtool/storage/local_ac.hpp" #include //std::ignore #include // std::move #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/local_ac.hpp" template auto LocalAC::StoreResult( ArtifactDigest const& action_id, bazel_re::ActionResult const& result) const noexcept -> bool { auto const cas_key = WriteAction(result); return cas_key.has_value() and WriteActionKey(action_id, *cas_key); } template auto LocalAC::CachedResult(ArtifactDigest const& action_id) const noexcept -> std::optional { auto const cas_key = ReadActionKey(action_id); if (not cas_key) { logger_->Emit(LogLevel::Debug, cas_key.error()); return std::nullopt; } auto result = ReadAction(*cas_key); if (not result) { logger_->Emit(LogLevel::Warning, "Parsing action result failed for action {}", action_id.hash()); return std::nullopt; } return std::move(result); } template template requires(kIsLocalGeneration) auto LocalAC::LocalUplinkEntry( LocalGenerationAC const& latest, ArtifactDigest const& action_id) const noexcept -> bool { // Determine action cache key path in latest generation. if (FileSystemManager::IsFile( latest.file_store_.GetPath(action_id.hash()))) { return true; } // Read cache key auto const cas_key = ReadActionKey(action_id); if (not cas_key) { return false; } // Read result (cache value) auto const result = ReadAction(*cas_key); if (not result) { return false; } // Uplink result content for (auto const& file : result->output_files()) { auto const digest = ArtifactDigestFactory::FromBazel( cas_.GetHashFunction().GetType(), file.digest()); if (not digest) { return false; } if (not cas_.LocalUplinkBlob( latest.cas_, *digest, file.is_executable())) { return false; } } for (auto const& link : result->output_file_symlinks()) { if (not cas_.LocalUplinkBlob( latest.cas_, ArtifactDigestFactory::HashDataAs( cas_.GetHashFunction(), link.target()), /*is_executable=*/false)) { return false; } } for (auto const& link : result->output_directory_symlinks()) { if (not cas_.LocalUplinkBlob( latest.cas_, ArtifactDigestFactory::HashDataAs( cas_.GetHashFunction(), link.target()), /*is_executable=*/false)) { return false; } } for (auto const& directory : result->output_directories()) { auto const digest = ArtifactDigestFactory::FromBazel( cas_.GetHashFunction().GetType(), directory.tree_digest()); if (not digest) { return false; } if (not cas_.LocalUplinkTree(latest.cas_, *digest)) { return false; } } // Uplink result (cache value) if (not cas_.LocalUplinkBlob(latest.cas_, *cas_key, /*is_executable=*/false)) { return false; } auto const ac_entry_path = file_store_.GetPath(action_id.hash()); // Uplink cache key return latest.file_store_.AddFromFile(action_id.hash(), ac_entry_path, /*is_owner=*/true); } template auto LocalAC::WriteActionKey( ArtifactDigest const& action_id, ArtifactDigest const& cas_key) const noexcept -> bool { nlohmann::json const content = {cas_key.hash(), cas_key.size()}; return file_store_.AddFromBytes(action_id.hash(), content.dump()); } template auto LocalAC::ReadActionKey(ArtifactDigest const& action_id) const noexcept -> expected { auto const key_path = file_store_.GetPath(action_id.hash()); if constexpr (kDoGlobalUplink) { // Uplink any existing action-cache entries in storage generations std::ignore = uplinker_.UplinkActionCacheEntry(action_id); } auto const key_content = FileSystemManager::ReadFile(key_path, ObjectType::File); if (not key_content) { return unexpected{ fmt::format("Cache miss, entry not found {}", key_path.string())}; } try { nlohmann::json const j = nlohmann::json::parse(*key_content); return ArtifactDigestFactory::Create(cas_.GetHashFunction().GetType(), j[0].template get(), j[1].template get(), /*is_tree=*/false); } catch (...) { return unexpected{fmt::format( "Parsing cache entry failed for action {}", action_id.hash())}; } } template auto LocalAC::WriteAction(bazel_re::ActionResult const& action) const noexcept -> std::optional { return cas_.StoreBlob(action.SerializeAsString(), /*is_executable=*/false); } template auto LocalAC::ReadAction(ArtifactDigest const& cas_key) const noexcept -> std::optional { auto const action_path = cas_.BlobPath(cas_key, /*is_executable=*/false); if (not action_path) { return std::nullopt; } auto const action_content = FileSystemManager::ReadFile(*action_path); if (not action_content) { return std::nullopt; } bazel_re::ActionResult action{}; if (action.ParseFromString(*action_content)) { return action; } return std::nullopt; } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/local_cas.hpp000066400000000000000000000434331516554100600265070ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_cas.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/large_object_cas.hpp" // IWYU pragma: keep #include "src/buildtool/storage/uplinker.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" /// \brief The local (logical) CAS for storing blobs and trees. /// Blobs can be stored/queried as executable or non-executable. Trees might be /// treated differently depending on the compatibility mode. Supports global /// uplinking across all generations. The uplink is automatically performed for /// every entry that is read and every entry that is stored and already exists /// in an older generation. /// \tparam kDoGlobalUplink Enable global uplinking. template class LocalCAS { public: /// Local CAS generation used by GC without global uplink. using LocalGenerationCAS = LocalCAS; /// \brief Create local CAS with base path. /// Note that the base path is concatenated by a single character /// 'f'/'x'/'t' for each internally used physical CAS. /// \param base The base path for the CAS. explicit LocalCAS( GenerationConfig const& config, gsl::not_null const*> const& uplinker) : cas_file_{config.storage_config->hash_function, config.cas_f, MakeUplinker(config, uplinker)}, cas_exec_{config.storage_config->hash_function, config.cas_x, MakeUplinker(config, uplinker)}, cas_tree_{config.storage_config->hash_function, config.cas_t, MakeUplinker(config, uplinker)}, cas_file_large_{this, config, uplinker}, cas_tree_large_{this, config, uplinker}, hash_function_{config.storage_config->hash_function} {} [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction { return hash_function_; } /// \brief Obtain path to the storage root. /// \param type Type of the storage to be obtained. /// \param large True if a large storage is needed. [[nodiscard]] auto StorageRoot(ObjectType type, bool large = false) const noexcept -> std::filesystem::path const& { if (large) { return IsTreeObject(type) ? cas_tree_large_.StorageRoot() : cas_file_large_.StorageRoot(); } switch (type) { case ObjectType::Tree: return cas_tree_.StorageRoot(); case ObjectType::Executable: return cas_exec_.StorageRoot(); default: return cas_file_.StorageRoot(); } } /// \brief Store blob from file path with x-bit. /// \tparam kOwner Indicates ownership for optimization (hardlink). /// \param file_path The path of the file to store as blob. /// \param is_executable Store blob with executable permissions. /// \returns Digest of the stored blob or nullopt otherwise. template [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path, bool is_executable) const noexcept -> std::optional { return is_executable ? cas_exec_.StoreBlobFromFile(file_path, kOwner) : cas_file_.StoreBlobFromFile(file_path, kOwner); } /// \brief Store blob from bytes with x-bit (default: non-executable). /// \param bytes The bytes to create the blob from. /// \param is_executable Store blob with executable permissions. /// \returns Digest of the stored blob or nullopt otherwise. [[nodiscard]] auto StoreBlob(std::string const& bytes, bool is_executable = false) const noexcept -> std::optional { return is_executable ? cas_exec_.StoreBlobFromBytes(bytes) : cas_file_.StoreBlobFromBytes(bytes); } /// \brief Store tree from file path. /// \tparam kOwner Indicates ownership for optimization (hardlink). /// \param file_path The path of the file to store as tree. /// \returns Digest of the stored tree or nullopt otherwise. template [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) const noexcept -> std::optional { return cas_tree_.StoreBlobFromFile(file_path, kOwner); } /// \brief Store tree from bytes. /// \param bytes The bytes to create the tree from. /// \returns Digest of the stored tree or nullopt otherwise. [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept -> std::optional { return cas_tree_.StoreBlobFromBytes(bytes); } /// \brief Obtain blob path from digest with x-bit. /// Performs a synchronization if blob is only available with inverse x-bit. /// \param digest Digest of the blob to lookup. /// \param is_executable Lookup blob with executable permissions. /// \returns Path to the blob if found or nullopt otherwise. [[nodiscard]] auto BlobPath(ArtifactDigest const& digest, bool is_executable) const noexcept -> std::optional { auto const path = BlobPathNoSync(digest, is_executable); return path ? path : TrySyncBlob(digest, is_executable); } /// \brief Obtain blob path from digest with x-bit. /// Synchronization between file/executable CAS is skipped. /// \param digest Digest of the blob to lookup. /// \param is_executable Lookup blob with executable permissions. /// \returns Path to the blob if found or nullopt otherwise. [[nodiscard]] auto BlobPathNoSync(ArtifactDigest const& digest, bool is_executable) const noexcept -> std::optional { return is_executable ? cas_exec_.BlobPath(digest) : cas_file_.BlobPath(digest); } /// \brief Split a blob into chunks. /// \param digest The digest of a blob to be split. /// \returns Digests of the parts of the large object or an /// error code on failure. [[nodiscard]] auto SplitBlob(ArtifactDigest const& digest) const noexcept -> expected, LargeObjectError> { return cas_file_large_.Split(digest); } /// \brief Splice a blob from parts. /// \param digest The expected digest of the result. /// \param parts The parts of the large object. /// \param is_executable Splice the blob with executable permissions. /// \return The digest of the result or an error code on /// failure. [[nodiscard]] auto SpliceBlob(ArtifactDigest const& digest, std::vector const& parts, bool is_executable) const noexcept -> expected { return is_executable ? Splice(digest, parts) : Splice(digest, parts); } /// \brief Obtain tree path from digest. /// \param digest Digest of the tree to lookup. /// \returns Path to the tree if found or nullopt otherwise. [[nodiscard]] auto TreePath(ArtifactDigest const& digest) const noexcept -> std::optional { return cas_tree_.BlobPath(digest); } /// \brief Split a tree into chunks. /// \param digest The digest of a tree to be split. /// \returns Digests of the parts of the large object or an /// error code on failure. [[nodiscard]] auto SplitTree(ArtifactDigest const& digest) const noexcept -> expected, LargeObjectError> { return cas_tree_large_.Split(digest); } /// \brief Splice a tree from parts. /// \param digest The expected digest of the result. /// \param parts The parts of the large object. /// \return The digest of the result or an error code on /// failure. [[nodiscard]] auto SpliceTree(ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected { return Splice(digest, parts); } /// \brief Check whether all parts of the tree are in the storage. /// \param tree_digest Digest of the tree to be checked. /// \param tree_data Content of the tree. /// \return An error on fail. [[nodiscard]] auto CheckTreeInvariant(ArtifactDigest const& tree_digest, std::string const& tree_data) const noexcept -> std::optional; /// \brief Check whether all parts of the tree are in the storage. /// \param tree_digest Digest of the tree to be checked. /// \param file Content of the tree. /// \return An error on fail. [[nodiscard]] auto CheckTreeInvariant(ArtifactDigest const& tree_digest, std::filesystem::path const& file) const noexcept -> std::optional; /// \brief Uplink blob from this generation to latest LocalCAS generation. /// Performs a synchronization if requested and if blob is only available /// with inverse x-bit. This function is only available for instances that /// are used as local GC generations (i.e., disabled global uplink). /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest LocalCAS generation. /// \param digest The digest of the blob to uplink. /// \param is_executable Uplink blob with executable permissions. /// \param skip_sync Do not sync between file/executable CAS. /// \param splice_result Create the result of splicing in the latest /// generation. /// \returns True if blob was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBlob( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool is_executable, bool skip_sync = false, bool splice_result = false) const noexcept -> bool; /// \brief Uplink tree from this generation to latest LocalCAS generation. /// This function is only available for instances that are used as local GC /// generations (i.e., disabled global uplink). Trees are uplinked deep, /// including all referenced entries. Note that in compatible mode we do not /// have the notion of "tree" and instead trees are stored as blobs. /// Therefore, in compatible mode this function is only used by instances /// that are aware of trees, such as output directories in action results or /// tree artifacts from target cache. /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest LocalCAS generation. /// \param digest The digest of the tree to uplink. /// \param splice_result Create the result of splicing in the latest /// generation. /// \returns True if tree was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkTree( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; /// \brief Uplink large entry from this generation to latest LocalCAS /// generation. This function is only available for instances that are used /// as local GC generations (i.e., disabled global uplink). /// \tparam kType Type of the large entry to be uplinked. /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest LocalCAS generation. /// \param latest_large The latest LargeObjectCAS /// \param digest The digest of the large entry to uplink. /// \returns True if the large entry was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkLargeObject( LocalGenerationCAS const& latest, ArtifactDigest const& digest) const noexcept -> bool; private: ObjectCAS cas_file_; ObjectCAS cas_exec_; ObjectCAS cas_tree_; LargeObjectCAS cas_file_large_; LargeObjectCAS cas_tree_large_; HashFunction hash_function_; /// \brief Provides uplink via "exists callback" for physical object CAS. template [[nodiscard]] static auto MakeUplinker( GenerationConfig const& config, gsl::not_null const*> const& uplinker) { if constexpr (kDoGlobalUplink) { bool const native = ProtocolTraits::IsNative( config.storage_config->hash_function.GetType()); return [native, uplinker](auto const& digest, auto const& /*path*/) { if constexpr (IsTreeObject(kType)) { // in non-compatible mode, do explicit deep tree uplink // in compatible mode, treat all trees as blobs if (native) { return uplinker->UplinkTree(digest); } } return uplinker->UplinkBlob(digest, IsExecutableObject(kType)); }; } else { return std::nullopt; } } /// \brief Try to sync blob between file CAS and executable CAS. /// \param digest Blob digest. /// \param to_executable Sync direction. /// \returns Path to blob in target CAS. [[nodiscard]] auto TrySyncBlob(ArtifactDigest const& digest, bool to_executable) const noexcept -> std::optional { auto const src_blob = BlobPathNoSync(digest, not to_executable); if (src_blob and StoreBlob(*src_blob, to_executable)) { return BlobPathNoSync(digest, to_executable); } return std::nullopt; } template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkGitTree( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, ArtifactDigest const& digest, gsl::not_null*> const& seen, bool splice_result = false) const noexcept -> bool; template requires(kIsLocalGeneration) [[nodiscard]] auto TrySplice(ArtifactDigest const& digest) const noexcept -> TmpFile::Ptr; template [[nodiscard]] auto Splice(ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected; }; #ifndef BOOTSTRAP_BUILD_TOOL // NOLINTNEXTLINE(misc-header-include-cycle) #include "src/buildtool/storage/local_cas.tpp" // IWYU pragma: export #else template auto LocalCAS::CheckTreeInvariant( ArtifactDigest const& tree_digest, std::string const& tree_data) const noexcept -> std::optional { return std::nullopt; } template auto LocalCAS::CheckTreeInvariant( ArtifactDigest const& tree_digest, std::filesystem::path const& file) const noexcept -> std::optional { return std::nullopt; } template template auto LocalCAS::Splice(ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected { return unexpected{ LargeObjectError{LargeObjectErrorCode::Internal, "not allowed"}}; } #endif #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/local_cas.tpp000066400000000000000000000374751516554100600265340ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP // IWYU pragma: private, include "src/buildtool/storage/local_cas.hpp" #include #include // std::move #include "fmt/core.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/local_cas.hpp" template template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkBlob( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool is_executable, bool skip_sync, bool splice_result) const noexcept -> bool { // Determine blob path in latest generation. auto blob_path_latest = latest.BlobPathNoSync(digest, is_executable); if (blob_path_latest) { return true; } // Determine blob path of given generation. auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable) : BlobPath(digest, is_executable); TmpFile::Ptr spliced; if (not blob_path) { spliced = TrySplice(digest); blob_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not blob_path) { return false; } if (spliced != nullptr) { // The result of uplinking of a large object must not affect the // result of uplinking in general. In other case, two sequential calls // to BlobPath might return different results: The first call splices // and uplinks the object, but fails at large entry uplinking. The // second call finds the tree in the youngest generation and returns. std::ignore = LocalUplinkLargeObject(latest, digest); if (not splice_result) { return true; } } // Uplink blob from older generation to the latest generation. if (spliced != nullptr and is_executable) { // During multithreaded splicing, the main process can be forked // (inheriting open file descriptors). In this case, an executable file // saved using hardlinking becomes inaccessible. To prevent this, // executables must be stored as copies made in a child process. return latest.StoreBlob(*blob_path, is_executable) .has_value(); } return latest.StoreBlob(*blob_path, is_executable) .has_value(); } template template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkTree( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { if (not ProtocolTraits::IsNative(hash_function_.GetType())) { std::unordered_set seen{}; return LocalUplinkBazelDirectory(latest, digest, &seen, splice_result); } return LocalUplinkGitTree(latest, digest, splice_result); } template template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkGitTree( LocalGenerationCAS const& latest, ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { // Determine tree path in latest generation. auto tree_path_latest = latest.cas_tree_.BlobPath(digest); if (tree_path_latest) { return true; } // Determine tree path of given generation. auto tree_path = cas_tree_.BlobPath(digest); TmpFile::Ptr spliced; if (not tree_path) { spliced = TrySplice(digest); tree_path = spliced != nullptr ? std::optional{spliced->GetPath()} : std::nullopt; } if (not tree_path) { return false; } // Determine tree entries. auto content = FileSystemManager::ReadFile(*tree_path); auto skip_symlinks = [](auto const& /*unused*/) { return true; }; auto tree_entries = GitRepo::ReadTreeData(*content, digest.hash(), skip_symlinks, /*is_hex_id=*/true); if (not tree_entries) { return false; } // Uplink tree entries. for (auto const& [raw_id, entry_vector] : *tree_entries) { // Process only first entry from 'entry_vector' since all // entries represent the same blob, just with different // names. auto const entry_type = entry_vector.front().type; auto const digest = ArtifactDigestFactory::Create(hash_function_.GetType(), ToHexString(raw_id), 0, IsTreeObject(entry_type)); if (not digest) { return false; } if (digest->IsTree()) { if (not LocalUplinkGitTree(latest, *digest)) { return false; } } else { if (not LocalUplinkBlob( latest, *digest, IsExecutableObject(entry_type))) { return false; } } } if (spliced != nullptr) { // Uplink the large entry afterwards: // The result of uplinking of a large object must not affect the // result of uplinking in general. In other case, two sequential calls // to TreePath might return different results: The first call splices // and uplinks the object, but fails at large entry uplinking. The // second call finds the tree in the youngest generation and returns. std::ignore = LocalUplinkLargeObject(latest, digest); if (not splice_result) { return true; } } // Uplink tree from older generation to the latest generation. return latest.cas_tree_.StoreBlobFromFile(*tree_path, /*is owner=*/true) .has_value(); } template template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, ArtifactDigest const& digest, gsl::not_null*> const& seen, bool splice_result) const noexcept -> bool { // Skip already uplinked directories if (seen->contains(digest)) { return true; } // Determine bazel directory path of given generation. auto dir_path = cas_tree_.BlobPath(digest); TmpFile::Ptr spliced; if (not dir_path) { spliced = TrySplice(digest); dir_path = spliced != nullptr ? std::optional{spliced->GetPath()} : std::nullopt; } if (not dir_path) { return false; } // Determine bazel directory entries. auto content = FileSystemManager::ReadFile(*dir_path); bazel_re::Directory dir{}; if (not dir.ParseFromString(*content)) { return false; } // Uplink bazel directory entries. for (auto const& file : dir.files()) { auto const digest = ArtifactDigestFactory::FromBazel( hash_function_.GetType(), file.digest()); if (not digest) { return false; } if (not LocalUplinkBlob(latest, *digest, file.is_executable())) { return false; } } for (auto const& directory : dir.directories()) { auto const digest = ArtifactDigestFactory::FromBazel( hash_function_.GetType(), directory.digest()); if (not digest) { return false; } if (not LocalUplinkBazelDirectory(latest, *digest, seen)) { return false; } } // Determine bazel directory path in latest generation. auto const dir_path_latest = latest.cas_tree_.BlobPath(digest); if (spliced != nullptr) { // Uplink the large entry afterwards: // The result of uplinking of a large object must not affect the // result of uplinking in general. In other case, two sequential // calls to TreePath might return different results: The first call // splices and uplinks the object, but fails at large entry // uplinking. The second call finds the tree in the youngest // generation and returns. std::ignore = LocalUplinkLargeObject(latest, digest); } bool const skip_store = spliced != nullptr and not splice_result; // Uplink bazel directory from older generation to the latest // generation. if (skip_store or dir_path_latest.has_value() or latest.cas_tree_.StoreBlobFromFile(*dir_path, /*is_owner=*/true)) { try { seen->emplace(digest); return true; } catch (...) { return false; } } return false; } template template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkLargeObject( LocalGenerationCAS const& latest, ArtifactDigest const& digest) const noexcept -> bool { if constexpr (IsTreeObject(kType)) { return cas_tree_large_.LocalUplink( latest, latest.cas_tree_large_, digest); } else { return cas_file_large_.LocalUplink( latest, latest.cas_file_large_, digest); } } template template requires(kIsLocalGeneration) auto LocalCAS::TrySplice( ArtifactDigest const& digest) const noexcept -> TmpFile::Ptr { auto spliced = IsTreeObject(kType) ? cas_tree_large_.TrySplice(digest) : cas_file_large_.TrySplice(digest); return spliced.has_value() ? spliced.value() : nullptr; } template auto LocalCAS::CheckTreeInvariant( ArtifactDigest const& tree_digest, std::string const& tree_data) const noexcept -> std::optional { if (not ProtocolTraits::IsNative(hash_function_.GetType())) { return std::nullopt; } auto skip_symlinks = [](auto const& /*unused*/) { return true; }; auto const entries = GitRepo::ReadTreeData(tree_data, tree_digest.hash(), skip_symlinks, /*is_hex_id=*/true); if (not entries) { return LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not read entries of the tree {}", tree_digest.hash())}; } // Ensure all entries are in the storage: for (const auto& entry : *entries) { for (auto const& item : entry.second) { auto const digest = ArtifactDigestFactory::Create(hash_function_.GetType(), ToHexString(entry.first), 0, // size unknown IsTreeObject(item.type)); if (not digest) { return LargeObjectError{ LargeObjectErrorCode::InvalidTree, fmt::format("tree invariant violated {}:\n {}", tree_digest.hash(), digest.error())}; } // To avoid splicing during search, large CASes are inspected first. bool const entry_exists = IsTreeObject(item.type) ? cas_tree_large_.GetEntryPath(*digest) or TreePath(*digest) : cas_file_large_.GetEntryPath(*digest) or BlobPath(*digest, IsExecutableObject(item.type)); if (not entry_exists) { return LargeObjectError{ LargeObjectErrorCode::InvalidTree, fmt::format("tree invariant violated {} : missing part {}", tree_digest.hash(), digest->hash())}; } } } return std::nullopt; } template auto LocalCAS::CheckTreeInvariant( ArtifactDigest const& tree_digest, std::filesystem::path const& file) const noexcept -> std::optional { auto const tree_data = FileSystemManager::ReadFile(file); if (not tree_data) { return LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not read tree {}", tree_digest.hash())}; } return CheckTreeInvariant(tree_digest, *tree_data); } template template auto LocalCAS::Splice(ArtifactDigest const& digest, std::vector const& parts) const noexcept -> expected { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); // Check file is spliced already: if (kIsTree ? TreePath(digest) : BlobPath(digest, kIsExec)) { return digest; } // Splice the result from parts: auto splice_result = kIsTree ? cas_tree_large_.Splice(digest, parts) : cas_file_large_.Splice(digest, parts); if (not splice_result) { return unexpected{std::move(splice_result).error()}; } auto const& large_object = *splice_result; // Check digest consistency: // Using Store{Tree, Blob} to calculate the resulting hash and later // decide whether the result is valid is unreasonable, because these // methods can refer to a file that existed before. The direct hash // calculation is done instead. auto const& file_path = large_object->GetPath(); auto spliced_digest = ArtifactDigestFactory::HashFileAs(hash_function_, file_path); if (not spliced_digest) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, "could not calculate digest"}}; } if (*spliced_digest != digest) { return unexpected{LargeObjectError{ LargeObjectErrorCode::InvalidResult, fmt::format("actual result {} differs from the expected one {}", spliced_digest->hash(), digest.hash())}}; } // Check tree invariants: if constexpr (kIsTree) { if (ProtocolTraits::IsNative(hash_function_.GetType())) { if (auto error = CheckTreeInvariant(digest, file_path)) { return unexpected{std::move(*error)}; } } } static constexpr bool kOwner = true; auto const stored_digest = kIsTree ? StoreTree(file_path) : StoreBlob(file_path, kIsExec); if (not stored_digest) { return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, fmt::format("could not splice {}", digest.hash())}}; } return *std::move(stored_digest); } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/repository_garbage_collector.cpp000066400000000000000000000130411516554100600325070ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/repository_garbage_collector.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" auto RepositoryGarbageCollector::SharedLock( StorageConfig const& storage_config) noexcept -> std::optional { return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/true); } auto RepositoryGarbageCollector::ExclusiveLock( StorageConfig const& storage_config) noexcept -> std::optional { return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/false); } auto RepositoryGarbageCollector::LockFilePath( StorageConfig const& storage_config) noexcept -> std::filesystem::path { return storage_config.RepositoryRoot() / "gc.lock"; } auto RepositoryGarbageCollector::TriggerGarbageCollection( StorageConfig const& storage_config, bool drop_only) noexcept -> bool { auto const remove_me_prefix = std::string{"remove-me"}; auto pid = CreateProcessUniqueId(); if (not pid) { return false; } auto remove_me = storage_config.RepositoryRoot() / (remove_me_prefix + *pid); // With a shared lock, we can remove that directory, if it exists, // as we own the process id. { auto lock = SharedLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to get a shared lock the for repository root"); return false; } if (FileSystemManager::IsDirectory(remove_me)) { if (not FileSystemManager::RemoveDirectory(remove_me)) { Logger::Log(LogLevel::Error, "Failed to remove directory {}", remove_me.string()); return false; } } else { if (not FileSystemManager::RemoveFile(remove_me)) { Logger::Log( LogLevel::Error, "Failed to remove {}", remove_me.string()); return false; } } } // after releasing the shared lock, wait to get an exclusive lock for doing // the critical renaming { auto lock = ExclusiveLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to exclusively lock the local repository root"); return false; } if (drop_only) { if (not FileSystemManager::CreateDirectory(remove_me)) { Logger::Log(LogLevel::Error, "Failed to create directory {}", remove_me.string()); return false; } for (std::size_t i = storage_config.num_generations - 1; i > 0; i--) { auto from = storage_config.RepositoryGenerationRoot(i); auto to = remove_me / (fmt::format("generation-{}", i)); if (FileSystemManager::IsDirectory(from)) { if (not FileSystemManager::Rename(from, to)) { Logger::Log(LogLevel::Error, "Failed to rename {} to {}", from.string(), to.string()); return false; } } } } else { for (std::size_t i = storage_config.num_generations; i > 0; i--) { auto from = storage_config.RepositoryGenerationRoot(i - 1); auto to = i < storage_config.num_generations ? storage_config.RepositoryGenerationRoot(i) : remove_me; if (FileSystemManager::IsDirectory(from)) { if (not FileSystemManager::Rename(from, to)) { Logger::Log(LogLevel::Error, "Failed to rename {} to {}", from.string(), to.string()); return false; } } } } } // Finally, with a shared lock, clean up the directory to be removed { auto lock = SharedLock(storage_config); if (not lock) { Logger::Log(LogLevel::Error, "Failed to get a shared lock for the repository root"); return false; } if (FileSystemManager::IsDirectory(remove_me)) { if (not FileSystemManager::RemoveDirectory(remove_me)) { Logger::Log(LogLevel::Error, "Failed to remove directory {}", remove_me.string()); return false; } } } return true; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/repository_garbage_collector.hpp000066400000000000000000000037201516554100600325170ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_REPOSITORY_GARBAGE_COLLECTOR_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_REPOSITORY_GARBAGE_COLLECTOR_HPP #include #include #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/file_locking.hpp" /// \brief Global garbage collector implementation. /// Responsible for deleting oldest generation. class RepositoryGarbageCollector { public: /// \brief Trigger garbage collection, i.e., rotate the generations and /// delete the oldest. \returns true on success. [[nodiscard]] auto static TriggerGarbageCollection( StorageConfig const& storage_config, bool drop_only = false) noexcept -> bool; /// \brief Acquire shared lock to prevent garbage collection from running. /// \param storage_config Storage to be locked. /// \returns The acquired lock file on success or nullopt otherwise. [[nodiscard]] auto static SharedLock( StorageConfig const& storage_config) noexcept -> std::optional; private: [[nodiscard]] auto static ExclusiveLock( StorageConfig const& storage_config) noexcept -> std::optional; [[nodiscard]] auto static LockFilePath( StorageConfig const& storage_config) noexcept -> std::filesystem::path; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/storage.hpp000066400000000000000000000071431516554100600262310ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP #include #include #include "gsl/gsl" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/local_ac.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/target_cache.hpp" #include "src/buildtool/storage/uplinker.hpp" #include "src/utils/cpp/gsl.hpp" /// \brief The local storage for accessing CAS and caches. /// Maintains an instance of LocalCAS, LocalAC, TargetCache. Supports global /// uplinking across all generations. The uplink is automatically performed by /// the affected storage instance (CAS, action cache, target cache). /// \tparam kDoGlobalUplink Enable global uplinking. template class LocalStorage final { public: static constexpr std::size_t kYoungest = 0U; using Uplinker_t = Uplinker; using CAS_t = LocalCAS; using AC_t = LocalAC; using TC_t = ::TargetCache; [[nodiscard]] static auto Create( gsl::not_null const& storage_config, std::size_t generation = kYoungest) -> LocalStorage { if constexpr (kDoGlobalUplink) { // It is not allowed to enable uplinking for old generations. EnsuresAudit(generation == kYoungest); } auto gen_config = storage_config->CreateGenerationConfig(generation); return LocalStorage{gen_config}; } [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction { return cas_->GetHashFunction(); } /// \brief Get the CAS instance. [[nodiscard]] auto CAS() const noexcept -> CAS_t const& { return *cas_; } /// \brief Get the action cache instance. [[nodiscard]] auto ActionCache() const noexcept -> AC_t const& { return *ac_; } /// \brief Get the target cache instance. [[nodiscard]] auto TargetCache() const noexcept -> TC_t const& { return *tc_; } private: std::unique_ptr uplinker_; std::unique_ptr cas_; std::unique_ptr ac_; std::unique_ptr tc_; explicit LocalStorage(GenerationConfig const& config) : uplinker_{std::make_unique(config.storage_config)}, cas_{std::make_unique(config, &*uplinker_)}, ac_{std::make_unique(&*cas_, config, &*uplinker_)}, tc_{std::make_unique(&*cas_, config, &*uplinker_)} {} }; #ifdef BOOTSTRAP_BUILD_TOOL // disable global uplinking constexpr auto kDefaultDoGlobalUplink = false; #else constexpr auto kDefaultDoGlobalUplink = true; #endif /// \brief Generation type, local storage without global uplinking. using Generation = LocalStorage; using Storage = LocalStorage; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/target_cache.hpp000066400000000000000000000215121516554100600271720ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/buildtool/storage/uplinker.hpp" /// \brief The high-level target cache for storing export target's data. /// Supports global uplinking across all generations. The uplink is /// automatically performed for every entry that is read and already exists in /// an older generation. /// \tparam kDoGlobalUplink Enable global uplinking. template class TargetCache { public: /// Local target cache generation used by GC without global uplink. using LocalGenerationTC = TargetCache; /// Callback type for downloading known artifacts to local CAS. using ArtifactDownloader = std::function const&)>; explicit TargetCache( gsl::not_null const*> const& cas, GenerationConfig const& config, gsl::not_null const*> const& uplinker) : TargetCache(cas, config.target_cache, uplinker, config.storage_config->backend_description) {} /// \brief Returns a new TargetCache backed by the same CAS, but the /// FileStorage uses the given \p backend_description 's hash. This is /// particularly useful for the just-serve server implementation, since the /// sharding must be performed according to the client's request and not /// following the server configuration. [[nodiscard]] auto WithShard(BackendDescription backend_description) const -> TargetCache { if (backend_description_ == backend_description) { return *this; } return TargetCache(&cas_, file_store_.StorageRoot().parent_path(), &uplinker_, std::move(backend_description)); } TargetCache(TargetCache const&) = default; TargetCache(TargetCache&&) noexcept = default; auto operator=(TargetCache const&) -> TargetCache& = default; auto operator=(TargetCache&&) noexcept -> TargetCache& = default; ~TargetCache() noexcept = default; /// \brief Store new key-entry pair in the target cache. /// \param key The target-cache key. /// \param value The target-cache value to store. /// \param downloader Callback for obtaining known artifacts to local CAS. /// \returns true on success. [[nodiscard]] auto Store( TargetCacheKey const& key, TargetCacheEntry const& value, ArtifactDownloader const& downloader) const noexcept -> bool; /// \brief Calculate TargetCacheKey based on auxiliary information. /// Doesn't create a TargetCacheEntry in the TargetCache. /// \return TargetCacheKey on success. [[nodiscard]] auto ComputeKey( ArtifactDigest const& repo_key, BuildMaps::Base::NamedTarget const& target_name, Configuration const& effective_config) const noexcept -> std::optional; /// \brief Read existing entry and object info from the target cache. /// \param key The target-cache key to read the entry from. /// \param shard Optional explicit shard, if the default is not intended. /// \returns Pair of cache entry and its object info on success or nullopt. [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept -> std::optional>; /// \brief Uplink entry from this to latest target cache generation. /// This function is only available for instances that are used as local GC /// generations (i.e., disabled global uplink). /// \tparam kIsLocalGeneration True if this instance is a local generation. /// \param latest The latest target cache generation. /// \param key The target-cache key for the entry to uplink. /// \returns True if entry was successfully uplinked. template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( LocalGenerationTC const& latest, TargetCacheKey const& key) const noexcept -> bool; private: // By default, overwrite existing entries. Unless this is a generation // (disabled global uplink), then we never want to overwrite any entries. static constexpr auto kStoreMode = kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; std::shared_ptr logger_{std::make_shared("TargetCache")}; LocalCAS const& cas_; FileStorage file_store_; Uplinker const& uplinker_; BackendDescription const backend_description_; explicit TargetCache( gsl::not_null const*> const& cas, std::filesystem::path const& root, gsl::not_null const*> const& uplinker, BackendDescription backend_description) : cas_{*cas}, file_store_{ root / backend_description.HashContent(cas->GetHashFunction()).hash()}, uplinker_{*uplinker}, backend_description_{std::move(backend_description)} { if constexpr (kDoGlobalUplink) { // Write backend description (which determines the target cache // shard) to CAS. It needs to be added for informational purposes // only, so it is not an error if insertion fails or returns an // unexpected result. auto const id = cas_.StoreBlob( backend_description_.GetDescription(), /*is_executable=*/false); auto const expected_hash = file_store_.StorageRoot().filename().string(); if (not id) { logger_->Emit(LogLevel::Debug, "TargetCache: Failed to add backend description " "{} to the storage:\n{}", expected_hash, backend_description_.GetDescription()); } else if (id->hash() != expected_hash) { logger_->Emit(LogLevel::Debug, "TargetCache: backend description was added to " "the storage with an unexpected hash. Expected " "{}, added with {}. Content:\n{}", expected_hash, id->hash(), backend_description_.GetDescription()); } } } template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( LocalGenerationTC const& latest, std::string const& key_digest) const noexcept -> bool; [[nodiscard]] auto DownloadKnownArtifacts( TargetCacheEntry const& value, ArtifactDownloader const& downloader) const noexcept -> bool; }; #ifdef BOOTSTRAP_BUILD_TOOL using ActiveTargetCache = TargetCache; #else // TargetCache type aware of bootstrapping using ActiveTargetCache = TargetCache; #endif // BOOTSTRAP_BUILD_TOOL // NOLINTNEXTLINE(misc-header-include-cycle) #include "src/buildtool/storage/target_cache.tpp" // IWYU pragma: export #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/target_cache.tpp000066400000000000000000000173131516554100600272120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP // IWYU pragma: private, include "src/buildtool/storage/target_cache.hpp" #include #include //std::ignore #include "nlohmann/json.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/storage/target_cache.hpp" template auto TargetCache::Store( TargetCacheKey const& key, TargetCacheEntry const& value, ArtifactDownloader const& downloader) const noexcept -> bool { if (not DownloadKnownArtifacts(value, downloader)) { return false; } if (auto digest = cas_.StoreBlob(value.ToJson().dump(2))) { auto data = Artifact::ObjectInfo{*digest, ObjectType::File}.ToString(); logger_->Emit(LogLevel::Debug, "Adding entry for key {} as {}", key.Id().ToString(), data); return file_store_.AddFromBytes(key.Id().digest.hash(), data); } return false; } template auto TargetCache::ComputeKey( ArtifactDigest const& repo_key, BuildMaps::Base::NamedTarget const& target_name, Configuration const& effective_config) const noexcept -> std::optional { try { // target's repository is content-fixed, we can compute a cache key auto target_desc = nlohmann::json{ {"repo_key", repo_key.hash()}, {"target_name", nlohmann::json{target_name.module, target_name.name}.dump()}, {"effective_config", effective_config.ToString()}}; if (auto target_key = cas_.StoreBlob(target_desc.dump(2), /*is_executable=*/false)) { return TargetCacheKey{{*target_key, ObjectType::File}}; } } catch (std::exception const& ex) { logger_->Emit(LogLevel::Error, "Creating target cache key failed with:\n{}", ex.what()); } return std::nullopt; } template auto TargetCache::Read( TargetCacheKey const& key) const noexcept -> std::optional> { auto id = key.Id().digest.hash(); auto entry_path = file_store_.GetPath(id); if constexpr (kDoGlobalUplink) { // Uplink any existing target cache entry in storage generations std::ignore = uplinker_.UplinkTargetCacheEntry(key, backend_description_); } auto const entry = FileSystemManager::ReadFile(entry_path, ObjectType::File); if (not entry) { logger_->Emit(LogLevel::Debug, "Cache miss, entry not found {}", entry_path.string()); return std::nullopt; } auto const hash_type = cas_.GetHashFunction().GetType(); if (auto info = Artifact::ObjectInfo::FromString(hash_type, *entry)) { if (auto path = cas_.BlobPath(info->digest, /*is_executable=*/false)) { if (auto value = FileSystemManager::ReadFile(*path)) { try { return std::make_pair( TargetCacheEntry{hash_type, nlohmann::json::parse(*value)}, std::move(*info)); } catch (std::exception const& ex) { logger_->Emit(LogLevel::Warning, "Parsing entry for key {} failed with:\n{}", key.Id().ToString(), ex.what()); } } } } logger_->Emit(LogLevel::Warning, "Reading entry for key {} failed", key.Id().ToString()); return std::nullopt; } template template requires(kIsLocalGeneration) auto TargetCache::LocalUplinkEntry( LocalGenerationTC const& latest, TargetCacheKey const& key) const noexcept -> bool { return LocalUplinkEntry(latest, key.Id().digest.hash()); } template template requires(kIsLocalGeneration) auto TargetCache::LocalUplinkEntry( LocalGenerationTC const& latest, std::string const& key_digest) const noexcept -> bool { // Determine target cache key path of given generation. if (FileSystemManager::IsFile(latest.file_store_.GetPath(key_digest))) { return true; } // Determine target cache entry location. auto cache_key = file_store_.GetPath(key_digest); auto raw_key = FileSystemManager::ReadFile(cache_key); if (not raw_key) { return false; } // Determine target cache entry location. auto entry_info = Artifact::ObjectInfo::FromString( cas_.GetHashFunction().GetType(), *raw_key); if (not entry_info) { return false; } // Determine target cache entry blob path of given generation. auto cache_entry = cas_.BlobPath(entry_info->digest, /*is_executable=*/false); if (not cache_entry) { return false; } // Determine artifacts referenced by target cache entry. auto raw_entry = FileSystemManager::ReadFile(*cache_entry); if (not raw_entry) { return false; } nlohmann::json json_desc{}; try { json_desc = nlohmann::json::parse(*raw_entry); } catch (std::exception const& ex) { return false; } auto entry = TargetCacheEntry::FromJson(cas_.GetHashFunction().GetType(), json_desc); // Uplink the implied export targets first for (auto const& implied_digest : entry.ToImplied()) { if (implied_digest != key_digest) { if (not LocalUplinkEntry(latest, implied_digest)) { return false; } } } std::vector artifacts_info; if (not entry.ToArtifacts(&artifacts_info)) { return false; } // Uplink referenced artifacts. for (auto const& info : artifacts_info) { if (info.type == ObjectType::Tree) { if (not cas_.LocalUplinkTree(latest.cas_, info.digest)) { return false; } } else if (not cas_.LocalUplinkBlob( latest.cas_, info.digest, IsExecutableObject(info.type))) { return false; } } // Uplink target cache entry blob. if (not cas_.LocalUplinkBlob(latest.cas_, entry_info->digest, /*is_executable=*/false)) { return false; } // Uplink target cache key return latest.file_store_.AddFromFile( key_digest, cache_key, /*is_owner=*/true); } template auto TargetCache::DownloadKnownArtifacts( TargetCacheEntry const& value, ArtifactDownloader const& downloader) const noexcept -> bool { std::vector artifacts_info; return downloader and value.ToArtifacts(&artifacts_info) and downloader(artifacts_info); } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/target_cache_entry.cpp000066400000000000000000000146221516554100600304120ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/target_cache_entry.hpp" #include #include #include #include #include #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/gsl.hpp" auto TargetCacheEntry::FromTarget( HashFunction::Type hash_type, AnalysedTargetPtr const& target, std::unordered_map const& replacements) noexcept -> std::optional { auto result = TargetResult{.artifact_stage = target->Artifacts(), .provides = target->Provides(), .runfiles = target->RunFiles()}; auto desc = result.ReplaceNonKnownAndToJson(replacements); if (not desc) { return std::nullopt; } std::vector implied{}; for (auto const& x : target->ImpliedExport()) { implied.emplace_back(x); } if (not implied.empty()) { (*desc)["implied export targets"] = implied; } return TargetCacheEntry{hash_type, *desc}; } auto TargetCacheEntry::FromJson(HashFunction::Type hash_type, nlohmann::json desc) noexcept -> TargetCacheEntry { return TargetCacheEntry(hash_type, std::move(desc)); } auto TargetCacheEntry::ToResult() const noexcept -> std::optional { return TargetResult::FromJson(hash_type_, desc_); } auto TargetCacheEntry::ToImplied() const noexcept -> std::set { std::set result{}; if (desc_.contains("implied export targets")) { try { for (auto const& x : desc_["implied export targets"]) { result.emplace(x); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Exception reading implied export targets: {}", ex.what()); } } return result; } auto TargetCacheEntry::ToImpliedIds(std::string const& entry_key_hash) const noexcept -> std::optional> { std::vector result{}; if (desc_.contains("implied export targets")) { try { for (auto const& x : desc_["implied export targets"]) { if (x != entry_key_hash) { auto digest = ArtifactDigestFactory::Create( hash_type_, x, 0, /*is_tree=*/false); if (not digest) { Logger::Log( LogLevel::Debug, "{}", std::move(digest).error()); return std::nullopt; } result.emplace_back( Artifact::ObjectInfo{.digest = *std::move(digest), .type = ObjectType::File}); } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Exception reading implied export targets: {}", ex.what()); return std::nullopt; } } return result; } [[nodiscard]] static auto ToObjectInfo(HashFunction::Type hash_type, nlohmann::json const& json) -> Artifact::ObjectInfo { auto const desc = ArtifactDescription::FromJson(hash_type, json); // The assumption is that all artifacts mentioned in a target cache // entry are KNOWN to the remote side. ExpectsAudit(desc and desc->IsKnown()); auto const& info = desc->ToArtifact().Info(); ExpectsAudit(info); return *info; } [[nodiscard]] static auto ScanArtifactMap( HashFunction::Type hash_type, gsl::not_null*> const& infos, nlohmann::json const& json) -> bool { if (not json.is_object()) { return false; } infos->reserve(infos->size() + json.size()); std::transform(json.begin(), json.end(), std::back_inserter(*infos), [hash_type](auto const& item) { return ToObjectInfo(hash_type, item); }); return true; } [[nodiscard]] static auto ScanProvidesMap( HashFunction::Type hash_type, gsl::not_null*> const& infos, nlohmann::json const& json) -> bool { if (not json.is_object()) { return false; } auto const& nodes = json["nodes"]; auto const& provided_artifacts = json["provided_artifacts"]; infos->reserve(infos->size() + provided_artifacts.size()); std::transform(provided_artifacts.begin(), provided_artifacts.end(), std::back_inserter(*infos), [hash_type, &nodes](auto const& item) { return ToObjectInfo( hash_type, nodes[item.template get()]); }); return true; } auto TargetCacheEntry::ToArtifacts( gsl::not_null*> const& infos) const noexcept -> bool { try { if (ScanArtifactMap(hash_type_, infos, desc_["artifacts"]) and ScanArtifactMap(hash_type_, infos, desc_["runfiles"]) and ScanProvidesMap(hash_type_, infos, desc_["provides"])) { return true; } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Scanning target cache entry for artifacts failed with:\n{}", ex.what()); } return false; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/target_cache_entry.hpp000066400000000000000000000062511516554100600304160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/crypto/hash_function.hpp" // Entry for target cache. Created from target, contains TargetResult. class TargetCacheEntry { public: explicit TargetCacheEntry(HashFunction::Type hash_type, nlohmann::json desc) : hash_type_{hash_type}, desc_(std::move(desc)) {} // Create the entry from target with replacement artifacts/infos. // Replacement artifacts must replace all non-known artifacts by known. [[nodiscard]] static auto FromTarget( HashFunction::Type hash_type, AnalysedTargetPtr const& target, std::unordered_map const& replacements) noexcept -> std::optional; // Create a target-cache entry from a json description. [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, nlohmann::json desc) noexcept -> TargetCacheEntry; // Obtain TargetResult from cache entry. [[nodiscard]] auto ToResult() const noexcept -> std::optional; // Obtain the implied export targets [[nodiscard]] auto ToImplied() const noexcept -> std::set; // Obtain the implied export targets hashes as a vector of ObjectInfo, // excluding the digest corresponding to this entry. As opposed to // ToImplied, it returns nullopt on exception. [[nodiscard]] auto ToImpliedIds(std::string const& entry_key_hash) const noexcept -> std::optional>; // Obtain all artifacts from cache entry (all should be known // artifacts). [[nodiscard]] auto ToArtifacts( gsl::not_null*> const& infos) const noexcept -> bool; [[nodiscard]] auto ToJson() const& -> nlohmann::json const& { return desc_; } [[nodiscard]] auto ToJson() && -> nlohmann::json { return std::move(desc_); } private: HashFunction::Type hash_type_; nlohmann::json desc_; }; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/target_cache_key.hpp000066400000000000000000000032501516554100600300410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP #include #include #include #include "src/buildtool/common/artifact.hpp" // Key for target cache. Created from target name and effective config. class TargetCacheKey { public: explicit TargetCacheKey(Artifact::ObjectInfo id) : id_{std::move(id)} {} [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& { return id_; } [[nodiscard]] auto Id() && -> Artifact::ObjectInfo { return std::move(id_); } [[nodiscard]] auto operator==(TargetCacheKey const& other) const -> bool { return this->Id() == other.Id(); } private: Artifact::ObjectInfo id_; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(TargetCacheKey const& key) const noexcept -> std::size_t { return std::hash{}(key.Id()); } }; } // namespace std #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/uplinker.cpp000066400000000000000000000111741516554100600264100ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/storage/uplinker.hpp" #include #include #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache.hpp" #include "src/buildtool/storage/target_cache_key.hpp" namespace { [[nodiscard]] auto CreateGenerations( gsl::not_null const& storage_config) noexcept -> std::vector { std::vector generations; generations.reserve(storage_config->num_generations); for (std::size_t i = 0; i < storage_config->num_generations; ++i) { generations.emplace_back(Generation::Create(storage_config, /*generation=*/i)); } return generations; } } // namespace GlobalUplinker::GlobalUplinker( gsl::not_null const& storage_config) noexcept : storage_config_{*storage_config}, generations_{CreateGenerations(&storage_config_)} {} GlobalUplinker::~GlobalUplinker() = default; auto GlobalUplinker::UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); return std::any_of( generations_.begin(), generations_.end(), [&latest, &digest, is_executable](Generation const& generation) { return generation.CAS().LocalUplinkBlob(latest, digest, is_executable, /*skip_sync=*/true, /*splice_result=*/true); }); } auto GlobalUplinker::UplinkTree(ArtifactDigest const& digest) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); return std::any_of(generations_.begin(), generations_.end(), [&latest, &digest](Generation const& generation) { return generation.CAS().LocalUplinkTree( latest, digest, /*splice_result=*/true); }); } auto GlobalUplinker::UplinkLargeBlob( ArtifactDigest const& digest) const noexcept -> bool { // Try to find large entry in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); return std::any_of( generations_.begin(), generations_.end(), [&latest, &digest](Generation const& generation) { return generation.CAS().LocalUplinkLargeObject( latest, digest); }); } auto GlobalUplinker::UplinkActionCacheEntry( ArtifactDigest const& action_id) const noexcept -> bool { // Try to find action-cache entry in all generations. auto const& latest = generations_[Generation::kYoungest].ActionCache(); return std::any_of(generations_.begin(), generations_.end(), [&latest, &action_id](Generation const& generation) { return generation.ActionCache().LocalUplinkEntry( latest, action_id); }); } auto GlobalUplinker::UplinkTargetCacheEntry( TargetCacheKey const& key, BackendDescription const& backend_description) const noexcept -> bool { // Try to find target-cache entry in all generations. auto const& latest = generations_[Generation::kYoungest].TargetCache().WithShard( backend_description); return std::any_of( generations_.begin(), generations_.end(), [&latest, &key, &backend_description](Generation const& generation) { return generation.TargetCache() .WithShard(backend_description) .LocalUplinkEntry(latest, key); }); } #endif // BOOTSTRAP_BUILD_TOOL just-buildsystem-justbuild-b1fb5fa/src/buildtool/storage/uplinker.hpp000066400000000000000000000105311516554100600264110ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_UPLINKER_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_UPLINKER_HPP #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/config.hpp" template class LocalStorage; // IWYU pragma: keep class TargetCacheKey; /// \brief Global uplinker implementation. /// Responsible for uplinking objects across all generations to latest /// generation. class GlobalUplinker final { public: explicit GlobalUplinker( gsl::not_null const& storage_config) noexcept; GlobalUplinker(GlobalUplinker const&) = default; GlobalUplinker(GlobalUplinker&&) noexcept = default; auto operator=(GlobalUplinker const&) = delete; auto operator=(GlobalUplinker&&) noexcept = delete; ~GlobalUplinker(); /// \brief Uplink blob across LocalCASes from all generations to latest. /// Note that blobs will NOT be synced between file/executable CAS. /// \param digest Digest of the blob to uplink. /// \param is_executable Indicate that blob is an executable. /// \returns true if blob was found and successfully uplinked. [[nodiscard]] auto UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool; /// \brief Uplink tree across LocalCASes from all generations to latest. /// Note that the tree will be deeply uplinked, i.e., all entries referenced /// by this tree will be uplinked before (including sub-trees). /// \param digest Digest of the tree to uplink. /// \returns true if tree was found and successfully uplinked (deep). [[nodiscard]] auto UplinkTree(ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink large blob entry across LocalCASes from all generations to /// latest. This method does not splice the large object. /// \param digest Digest of the large blob entry to uplink. /// \returns true if large entry was found and successfully uplinked. [[nodiscard]] auto UplinkLargeBlob( ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink entry from action cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. /// \param action_id Id of the action to uplink entry for. /// \returns true if cache entry was found and successfully uplinked. [[nodiscard]] auto UplinkActionCacheEntry( ArtifactDigest const& action_id) const noexcept -> bool; /// \brief Uplink entry from target cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. /// \param key Target cache key to uplink entry for. /// \param backend_description Explicit backend description. /// \returns true if cache entry was found and successfully uplinked. [[nodiscard]] auto UplinkTargetCacheEntry( TargetCacheKey const& key, BackendDescription const& backend_description) const noexcept -> bool; private: StorageConfig const& storage_config_; std::vector> const generations_; }; /// \brief An empty constructable Uplinker. Although it doesn't have any /// interface, it allows objects employing uplinking store the uplinker by /// reference instead of unobvious 'optional' raw pointers. class StubUplinker final { public: explicit StubUplinker( gsl::not_null const& /*unused*/) noexcept {} }; template using Uplinker = std::conditional_t; #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_UPLINKER_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/system/000077500000000000000000000000001516554100600237275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/system/TARGETS000066400000000000000000000014331516554100600247640ustar00rootroot00000000000000{ "system": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["VALGRIND_BUILD"] , "name": ["system"] , "hdrs": ["system.hpp"] , "srcs": ["system.cpp"] , "private-defines": { "type": "if" , "cond": {"type": "var", "name": "VALGRIND_BUILD"} , "then": ["VALGRIND_BUILD"] } , "deps": [["src/utils/cpp", "concepts"]] , "stage": ["src", "buildtool", "system"] } , "system_command": { "type": ["@", "rules", "CC", "library"] , "name": ["system_command"] , "hdrs": ["system_command.hpp"] , "deps": [ "system" , ["@", "gsl", "", "gsl"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] , "stage": ["src", "buildtool", "system"] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/system/system.cpp000066400000000000000000000030211516554100600257530ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/system/system.hpp" #ifdef VALGRIND_BUILD #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif // __unix__ #include #include #else #include #endif // VALGRIND_BUILD void System::ExitWithoutCleanup(int exit_code) { #ifdef VALGRIND_BUILD // Usually std::_Exit() is the right thing to do in child processes that do // not need to perform any cleanup (static destructors etc.). However, // Valgrind will trace child processes until exec(3) is called or otherwise // complains about leaks. Therefore, exit child processes via execvpe(3) if // VALGRIND_BUILD is defined. auto cmd = std::string{exit_code == EXIT_SUCCESS ? "/bin/true" : "/bin/false"}; auto args = std::array{cmd.data(), nullptr}; ::execvpe(args[0], args.data(), nullptr); #else std::_Exit(exit_code); #endif // VALGRIND_BUILD } just-buildsystem-justbuild-b1fb5fa/src/buildtool/system/system.hpp000066400000000000000000000047261516554100600257750ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_HPP #define INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_HPP #include #include "src/utils/cpp/concepts.hpp" namespace System { void ExitWithoutCleanup(int exit_code); /// \brief Obtain POSIX epoch time for a given clock. /// Clocks may have different epoch times. To obtain the POSIX epoch time /// (1970-01-01 00:00:00 UTC) for a given clock, it must be converted. template static auto GetPosixEpoch() -> std::chrono::time_point { // Since C++20, the system clock's default value is the POSIX epoch time. std::chrono::time_point sys_epoch{}; if constexpr (std::is_same_v) { // No conversion necessary for the system clock. return sys_epoch; } else if constexpr (ClockHasFromSys) { // The correct C++20 way to perform the time point conversion. return TClock::from_sys(sys_epoch); } else if constexpr (ClockHasFromTime) { // Older releases of libcxx did not implement the standard conversion // function from_sys() for std::chrono::file_clock. Instead the // non-standard function file_clock::from_time_t() must be used. Since // libcxx 14.0.0, this has been fixed by: // - https://reviews.llvm.org/D113027 // - https://reviews.llvm.org/D113430 // TODO(modernize): remove this once we require clang version >= 14.0.0 return TClock::from_time_t( std::chrono::system_clock::to_time_t(sys_epoch)); } static_assert(std::is_same_v or ClockHasFromSys or ClockHasFromTime, "Time point conversion function unavailable."); return {}; } } // namespace System #endif // INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/system/system_command.hpp000066400000000000000000000224021516554100600274620ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_COMMAND_HPP #define INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_COMMAND_HPP #ifdef __unix__ #include #include #include #else #error "Non-unix is not supported yet" #endif #include // for transform #include // for errno #include #include // for EXIT_FAILURE, WEXITSTATUS, WIFEXITED, WIFSIGNALED, WTERMSIG #include // for strerror() #include // for path, operator/ #include #include #include #include #include #include // std::move #include #include "gsl/gsl" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/system/system.hpp" /// \brief Execute system commands and obtain stdout, stderr and return value. /// Subsequent commands are context free and are not affected by previous /// commands. This class is not thread-safe. class SystemCommand { public: /// \brief Create execution system with name. explicit SystemCommand(std::string name) : logger_{std::move(name)} {} /// \brief Execute command and arguments. /// Stdout and stderr can be read from files named 'stdout' and 'stderr' /// created in `outdir`. Those files must not exist before the execution. /// \param argv argv vector with the command to execute /// \param env Environment variables set for execution. /// \param cwd Working directory for execution. /// \param outdir Directory for storing stdout/stderr files. /// \returns The command's exit code, or std::nullopt on execution error. [[nodiscard]] auto Execute(std::vector argv, std::map env, std::filesystem::path const& cwd, std::filesystem::path const& outdir) noexcept -> std::optional { if (not FileSystemManager::IsDirectory(outdir)) { logger_.Emit(LogLevel::Error, "Output directory does not exist {}", outdir.string()); return std::nullopt; } if (argv.empty()) { logger_.Emit(LogLevel::Error, "Command cannot be empty."); return std::nullopt; } std::vector cmd = UnwrapStrings(&argv); std::vector env_string{}; std::transform(std::begin(env), std::end(env), std::back_inserter(env_string), [](auto& name_value) { return name_value.first + "=" + name_value.second; }); std::vector envp = UnwrapStrings(&env_string); return ExecuteCommand(cmd.data(), envp.data(), cwd, outdir); } private: Logger logger_; /// \brief Open file exclusively as write-only. [[nodiscard]] static auto OpenFile( std::filesystem::path const& file_path) noexcept { static auto file_closer = [](gsl::owner f) { if (f != nullptr) { std::fclose(f); } }; return std::unique_ptr( std::fopen(file_path.c_str(), "wx"), file_closer); } /// \brief Execute command and arguments. /// \param cmd Command arguments as char pointer array. /// \param envp Environment variables as char pointer array. /// \param cwd Working directory for execution. /// \param outdir Directory for storing stdout/stderr files. /// \returns ExecOutput if command was successfully submitted to the system. /// \returns std::nullopt on internal failure. [[nodiscard]] auto ExecuteCommand( char* const* cmd, char* const* envp, std::filesystem::path const& cwd, std::filesystem::path const& outdir) noexcept -> std::optional { auto stdout_file = outdir / "stdout"; auto stderr_file = outdir / "stderr"; if (auto const out = OpenFile(stdout_file)) { if (auto const err = OpenFile(stderr_file)) { if (auto retval = ForkAndExecute( cmd, envp, cwd, fileno(out.get()), fileno(err.get()))) { return retval; } } else { logger_.Emit(LogLevel::Error, "Failed to open stderr file '{}' with error: {}", stderr_file.string(), strerror(errno)); } } else { logger_.Emit(LogLevel::Error, "Failed to open stdout file '{}' with error: {}", stdout_file.string(), strerror(errno)); } return std::nullopt; } /// \brief Fork process and exec command. /// \param cmd Command arguments as char pointer array. /// \param envp Environment variables as char pointer array. /// \param cwd Working directory for execution. /// \param out_fd File descriptor to standard output file. /// \param err_fd File descriptor to standard erro file. /// \returns return code if command was successfully submitted to system. /// \returns std::nullopt if fork or exec failed. [[nodiscard]] auto ForkAndExecute(char* const* cmd, char* const* envp, std::filesystem::path const& cwd, int out_fd, int err_fd) const noexcept -> std::optional { auto const* cwd_cstr = cwd.c_str(); // some executables require an open (possibly seekable) stdin, and // therefore, we use an open temporary file that does not appear on the // file system and will be removed automatically once the descriptor is // closed. gsl::owner in_file = std::tmpfile(); auto in_fd = fileno(in_file); // fork child process pid_t pid = ::fork(); if (-1 == pid) { logger_.Emit(LogLevel::Error, "Failed to execute '{}': cannot fork a child process.", *cmd); return std::nullopt; } // dispatch child/parent process if (pid == 0) { ::chdir(cwd_cstr); // redirect and close fds ::dup2(in_fd, STDIN_FILENO); ::dup2(out_fd, STDOUT_FILENO); ::dup2(err_fd, STDERR_FILENO); ::close(in_fd); ::close(out_fd); ::close(err_fd); // execute command in child process and exit ::execvpe(*cmd, cmd, envp); // report error and terminate child process if ::execvp did not exit // NOLINTNEXTLINE printf("Failed to execute '%s' with error: %s\n", *cmd, strerror(errno)); System::ExitWithoutCleanup(EXIT_FAILURE); } ::close(in_fd); // wait for child to finish and obtain return value int status{}; std::optional retval{std::nullopt}; while (not retval) { if (::waitpid(pid, &status, 0) == -1) { // this should never happen logger_.Emit(LogLevel::Error, "Waiting for child failed with: {}", strerror(errno)); break; } if (WIFEXITED(status)) { // NOLINT(hicpp-signed-bitwise) retval = WEXITSTATUS(status); // NOLINT(hicpp-signed-bitwise) } else if (WIFSIGNALED(status)) { // NOLINT(hicpp-signed-bitwise) constexpr auto kSignalBit = 128; auto sig = WTERMSIG(status); // NOLINT(hicpp-signed-bitwise) retval = kSignalBit + sig; logger_.Emit( LogLevel::Debug, "Child got killed by signal {}", sig); } // continue waitpid() in case we got STOPSIG from child } return retval; } static auto UnwrapStrings(std::vector* v) noexcept -> std::vector { std::vector raw{}; std::transform(std::begin(*v), std::end(*v), std::back_inserter(raw), [](auto& str) { return str.data(); }); raw.push_back(nullptr); return raw; } }; #endif // INCLUDED_SRC_BUILDTOOL_SYSTEM_SYSTEM_COMMAND_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/000077500000000000000000000000001516554100600254625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/TARGETS000066400000000000000000000035651516554100600265270ustar00rootroot00000000000000{ "tree_structure_cache": { "type": ["@", "rules", "CC", "library"] , "name": ["tree_structure_cache"] , "hdrs": ["tree_structure_cache.hpp"] , "srcs": ["tree_structure_cache.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "common"] , ["src/buildtool/file_system", "file_storage"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "buildtool", "tree_structure"] , "private-deps": [ ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] } , "tree_structure_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["tree_structure_utils"] , "hdrs": ["tree_structure_utils.hpp"] , "srcs": ["tree_structure_utils.cpp"] , "deps": [ "tree_structure_cache" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "buildtool", "tree_structure"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/git", "git_api"] , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "tmp_dir"] ] } } just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/tree_structure_cache.cpp000066400000000000000000000122101516554100600323640ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/tree_structure/tree_structure_cache.hpp" #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" namespace { inline constexpr std::size_t kHash = 0; inline constexpr std::size_t kSize = 1; inline constexpr std::size_t kTree = 2; [[nodiscard]] auto IsInCas(Storage const& storage, ArtifactDigest const& digest) -> bool { if (digest.IsTree()) { return storage.CAS().TreePath(digest).has_value(); } static constexpr bool kIsExecutable = true; return storage.CAS().BlobPathNoSync(digest, kIsExecutable).has_value() or storage.CAS().BlobPathNoSync(digest, not kIsExecutable).has_value(); } [[nodiscard]] auto ToJson(ArtifactDigest const& digest) -> std::optional { try { return nlohmann::json{digest.hash(), digest.size(), digest.IsTree()}; } catch (...) { return std::nullopt; } } [[nodiscard]] auto Parse(HashFunction::Type hash_type, std::filesystem::path const& path) noexcept -> std::optional { try { std::ifstream stream(path); nlohmann::json j = nlohmann::json::parse(stream); auto result = ArtifactDigestFactory::Create( hash_type, j.at(kHash).template get(), j.at(kSize).template get(), j.at(kTree).template get()); if (result) { return *std::move(result); } } catch (...) { return std::nullopt; } return std::nullopt; } } // namespace TreeStructureCache::TreeStructureCache( gsl::not_null const& storage_config) noexcept : TreeStructureCache(storage_config, 0, /*uplink=*/true) {} TreeStructureCache::TreeStructureCache( gsl::not_null const& storage_config, std::size_t generation, bool uplink) noexcept : storage_config_{*storage_config}, file_storage_{storage_config_.RepositoryGenerationRoot(generation) / "tree_structure"}, uplink_{uplink} {} auto TreeStructureCache::Get(ArtifactDigest const& key) const noexcept -> std::optional { // Check key object is in the storage AND trigger uplinking if needed auto const path = file_storage_.GetPath(key.hash()); if (uplink_ and not LocalUplinkObject(key, *this)) { return std::nullopt; } if (not FileSystemManager::IsFile(path)) { return std::nullopt; } return Parse(storage_config_.hash_function.GetType(), path); } auto TreeStructureCache::Set(ArtifactDigest const& key, ArtifactDigest const& value) const noexcept -> bool { auto result = Get(key); if (result) { return *result == value; } auto const storage = Storage::Create(&storage_config_); // Check both key and value are in the storage and in the latest generation. if (not IsInCas(storage, key) or not IsInCas(storage, value)) { return false; } try { auto j = ToJson(value); if (not j) { return false; } return file_storage_.AddFromBytes(key.hash(), j->dump()); } catch (...) { return false; } } auto TreeStructureCache::LocalUplinkObject( ArtifactDigest const& key, TreeStructureCache const& latest) const noexcept -> bool { auto const storage = Storage::Create(&storage_config_); // ensure key is present in the storage AND is in the latest generation if (not IsInCas(storage, key)) { return false; } for (std::size_t i = 0; i < storage_config_.num_generations; ++i) { TreeStructureCache const generation_cache(&storage_config_, i, false); auto const path = generation_cache.file_storage_.GetPath(key.hash()); if (not FileSystemManager::IsFile(path)) { continue; } auto digest = Parse(storage_config_.hash_function.GetType(), path); // ensure value is present in the storage AND is in the latest // generation if (not digest or not IsInCas(storage, *digest)) { return false; } static constexpr bool kIsOwner = true; return latest.file_storage_.AddFromFile(key.hash(), path, kIsOwner); } return false; } just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/tree_structure_cache.hpp000066400000000000000000000050541516554100600324010ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP #define INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" class TreeStructureCache final { public: explicit TreeStructureCache( gsl::not_null const& storage_config) noexcept; /// \brief Obtain the digest describing the tree structure of a key tree. /// Can trigger deep uplinking of referenced objects (both key and value). [[nodiscard]] auto Get(ArtifactDigest const& key) const noexcept -> std::optional; /// \brief Set coupling between key and value digest signalizing that the /// value digest contains the tree structure of a key digest. Both key and /// value are expected to be in the storage. Can trigger deep uplinking of /// objects (both key and value). /// \return True if the cache contains the key-value coupling. Fails if /// there is the key in the storage, but it refers to another value. Fails /// if key or value aren't present in the storage. [[nodiscard]] auto Set(ArtifactDigest const& key, ArtifactDigest const& value) const noexcept -> bool; private: StorageConfig const& storage_config_; FileStorage file_storage_; bool uplink_; explicit TreeStructureCache( gsl::not_null const& storage_config, std::size_t generation, bool uplink) noexcept; [[nodiscard]] auto LocalUplinkObject( ArtifactDigest const& key, TreeStructureCache const& latest) const noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/tree_structure_utils.cpp000066400000000000000000000267021516554100600324740ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/tree_structure/tree_structure_utils.hpp" #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto TreeStructureUtils::Compute(ArtifactDigest const& tree, Storage const& storage, TreeStructureCache const& cache) noexcept -> expected { if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) { return unexpected{fmt::format("Not a git tree: {}", tree.hash())}; } if (auto result = cache.Get(tree)) { return *std::move(result); } auto const tree_path = storage.CAS().TreePath(tree); if (not tree_path) { return unexpected{ fmt::format("Failed to read from the storage: {}", tree.hash())}; } auto const tree_content = FileSystemManager::ReadFile(*tree_path); if (not tree_content) { return unexpected{ fmt::format("Failed to read content of: {}", tree.hash())}; } auto skip_symlinks = [](auto const& /*unused*/) { return true; }; auto const entries = GitRepo::ReadTreeData( *tree_content, tree.hash(), skip_symlinks, /*is_hex_id=*/true); if (not entries) { return unexpected{ fmt::format("Failed to parse git tree: {}", tree.hash())}; } GitRepo::tree_entries_t structure_entries{}; for (auto const& [raw_id, es] : *entries) { for (auto const& entry : es) { std::optional structure_digest; if (IsTreeObject(entry.type)) { auto const git_digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, ToHexString(raw_id), /*size is unknown*/ 0, /*is_tree=*/true); if (not git_digest) { return unexpected{git_digest.error()}; } auto sub_tree = Compute(*git_digest, storage, cache); if (not sub_tree) { return sub_tree; } structure_digest = *std::move(sub_tree); } else { structure_digest = storage.CAS().StoreBlob( std::string{}, IsExecutableObject(entry.type)); } if (not structure_digest) { return unexpected{fmt::format( "Failed to get structure digest for: {}", raw_id)}; } if (auto id = FromHexString(structure_digest->hash())) { structure_entries[*std::move(id)].emplace_back(entry); } else { return unexpected{ fmt::format("Failed to get raw id for {}", raw_id)}; } } } auto const structure_tree = GitRepo::CreateShallowTree(structure_entries); if (not structure_tree) { return unexpected{fmt::format( "Failed to create structured Git tree for {}", tree.hash())}; } auto tree_structure = storage.CAS().StoreTree(structure_tree->second); if (not tree_structure) { return unexpected{fmt::format( "Failed to add tree structure to the CAS for {}", tree.hash())}; } if (not cache.Set(tree, *tree_structure)) { return unexpected{fmt::format( "Failed to create a tree structure cache entry for\n{} => {}", tree.hash(), tree_structure->hash())}; } return *std::move(tree_structure); } auto TreeStructureUtils::ImportToGit( ArtifactDigest const& tree, IExecutionApi const& source_api, StorageConfig const& target_config, gsl::not_null const& tagging_lock) noexcept -> expected { if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) { return unexpected{fmt::format("Not a git tree: {}", tree.hash())}; } // Check the source contains the tree: if (not source_api.IsAvailable(tree)) { return unexpected{ fmt::format("Source doesn't contain tree {}", tree.hash())}; } // Check the tree is in the repository already: if (auto in_repo = GitRepo::IsTreeInRepo(target_config.GitRoot(), tree.hash())) { if (*in_repo) { return tree; } } else { return unexpected{std::move(in_repo).error()}; } auto const tmp_dir = target_config.CreateTypedTmpDir("import_from_cas_to_git"); if (tmp_dir == nullptr) { return unexpected{fmt::format( "Failed to create temporary directory for {}", tree.hash())}; } // Stage the tree to a temporary directory: if (not source_api.RetrieveToPaths( {Artifact::ObjectInfo{tree, ObjectType::Tree}}, {tmp_dir->GetPath()})) { return unexpected{fmt::format( "Failed to stage {} to a temporary location.", tree.hash())}; } // Import the result to git: auto tree_hash = GitRepo::ImportToGit( target_config, tmp_dir->GetPath(), /*commit_message=*/fmt::format("Import {}", tree.hash()), tagging_lock); if (not tree_hash) { return unexpected{std::move(tree_hash).error()}; } auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, tree_hash.value(), /*size_unknown=*/0, /*is_tree=*/true); if (not digest) { return unexpected{std::move(digest).error()}; } return *digest; } auto TreeStructureUtils::ExportFromGit( ArtifactDigest const& tree, std::vector const& source_repos, StorageConfig const& storage_config, IExecutionApi const& target_api) noexcept -> expected { if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) { return unexpected{fmt::format("Not a git tree: {}", tree.hash())}; } // Find git repo that contains the tree: std::filesystem::path const* repo = nullptr; for (std::size_t i = 0; i < source_repos.size() and repo == nullptr; ++i) { auto in_repo = GitRepo::IsTreeInRepo(source_repos[i], tree.hash()); if (not in_repo.has_value()) { return unexpected{std::move(in_repo).error()}; } if (*in_repo) { repo = &source_repos[i]; } } // Check that at least one repo contains the tree: if (repo == nullptr) { return false; } RepositoryConfig repo_config{}; if (not repo_config.SetGitCAS(*repo, &storage_config)) { return unexpected{ fmt::format("Failed to set git cas at {}", repo->string())}; } auto const git_api = GitApi{&repo_config}; return git_api.RetrieveToCas({Artifact::ObjectInfo{tree, ObjectType::Tree}}, target_api); } auto TreeStructureUtils::ComputeStructureLocally( ArtifactDigest const& tree, std::vector const& known_repositories, StorageConfig const& storage_config, gsl::not_null const& tagging_lock) -> expected, std::string> { if (not ProtocolTraits::IsNative(tree.GetHashType()) or not tree.IsTree()) { return unexpected{fmt::format("Not a git tree: {}", tree.hash())}; } if (not ProtocolTraits::IsNative(storage_config.hash_function.GetType())) { return unexpected{fmt::format("Not a native storage config")}; } auto const storage = Storage::Create(&storage_config); LocalExecutionConfig const dummy_exec_config{}; LocalContext const local_context{.exec_config = &dummy_exec_config, .storage_config = &storage_config, .storage = &storage}; LocalApi const local_api{&local_context}; // First check the result is in cache already: TreeStructureCache const tree_structure_cache{&storage_config}; if (auto const from_cache = tree_structure_cache.Get(tree)) { auto to_git = ImportToGit(*from_cache, local_api, storage_config, tagging_lock); if (not to_git.has_value()) { return unexpected{fmt::format("While importing {} to git:\n{}", from_cache->hash(), std::move(to_git).error())}; } return std::make_optional(std::move(to_git).value()); } // If the tree is not in the storage, it must be present in git: if (not storage.CAS().TreePath(tree).has_value()) { auto in_cas = ExportFromGit(tree, known_repositories, storage_config, local_api); if (not in_cas.has_value()) { return unexpected{ fmt::format("While exporting {} from git to CAS:\n{}", tree.hash(), std::move(in_cas).error())}; } // If the tree hasn't been found neither in CAS, nor in git, there's // nothing else to do: if (not storage.CAS().TreePath(tree).has_value()) { return std::optional{}; } } // Compute tree structure and add it to the storage and cache: auto tree_structure = Compute(tree, storage, tree_structure_cache); if (not tree_structure) { return unexpected{ fmt::format("Failed to compute tree structure of {}:\n{}", tree.hash(), std::move(tree_structure).error())}; } // Import the result to git: auto to_git = ImportToGit(*tree_structure, local_api, storage_config, tagging_lock); if (not to_git.has_value()) { return unexpected{fmt::format( "While importing the resulting tree structure {} to git:\n{}", tree_structure->hash(), std::move(to_git).error())}; } return std::make_optional(*std::move(tree_structure)); } just-buildsystem-justbuild-b1fb5fa/src/buildtool/tree_structure/tree_structure_utils.hpp000066400000000000000000000112601516554100600324720ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP #define INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/tree_structure/tree_structure_cache.hpp" #include "src/utils/cpp/expected.hpp" class TreeStructureUtils final { public: /// \brief Compute the tree structure of a git tree and add corresponding /// coupling to the cache. Tree structure is a directory, where all blobs /// and symlinks are replaced with empty blobs. Every subtree gets written /// to the cache as well. Expects tree is present in the storage. /// \param tree Git tree to be analyzed. Must be present in the /// storage. /// \param storage Storage (GitSHA1) to be used for adding new tree /// structure artifacts /// \param cache Cache for storing key-value dependencies. /// \return Digest of the tree structure that is present in the storage on /// success, or an error message on failure. [[nodiscard]] static auto Compute(ArtifactDigest const& tree, Storage const& storage, TreeStructureCache const& cache) noexcept -> expected; /// \brief Import a git tree from the given source to storage_config's git /// repo. /// \param tree GitSHA1 tree to import /// \param source_api The source of the tree. Must be capable of /// processing GitSHA1 trees. /// \param target_config Config with target git repo. /// \param tagging_lock Mutex to protect critical tagging operation /// \return Digest of the tree that is available in the repo after the /// call or an error message on failure. [[nodiscard]] static auto ImportToGit( ArtifactDigest const& tree, IExecutionApi const& source_api, StorageConfig const& target_config, gsl::not_null const& tagging_lock) noexcept -> expected; /// \brief Export a tree from source git repositories to target api. Uses /// regular GitApi for retrieval from git and doesn't perform rehashing. /// \param tree Tree to export /// \param source_repos Repositories to check /// \param storage_config Storage to use for lookups /// \param target_api Api to export the tree to /// \return True if target api contains tree after the call, false if none /// of source repositories contain tree, or an error message on failure. [[nodiscard]] static auto ExportFromGit( ArtifactDigest const& tree, std::vector const& source_repos, StorageConfig const& storage_config, IExecutionApi const& target_api) noexcept -> expected; /// \brief Find git tree locally and compute its tree structure. /// \param tree Git tree to process /// \param known_repositories Known git repositories to check /// \param storage_config Storage to use for lookup and import. /// \param tagging_lock Mutex to protect critical git operations /// \return Digest of the tree structure that is available in /// storage_config's git repo and in storage_config's CAS; std::nullopt if /// the search failed to locate the tree's sources locally; an error string /// on critical failure. [[nodiscard]] static auto ComputeStructureLocally( ArtifactDigest const& tree, std::vector const& known_repositories, StorageConfig const& storage_config, gsl::not_null const& tagging_lock) -> expected, std::string>; }; #endif // INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/000077500000000000000000000000001516554100600227475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/000077500000000000000000000000001516554100600257755ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/TARGETS000066400000000000000000000036451516554100600270410ustar00rootroot00000000000000{ "git_ops_types": { "type": ["@", "rules", "CC", "library"] , "name": ["git_ops_types"] , "hdrs": ["git_ops_types.hpp"] , "deps": [["src/buildtool/file_system", "git_cas"], ["src/utils/cpp", "path"]] , "stage": ["src", "other_tools", "git_operations"] } , "git_operations": { "type": ["@", "rules", "CC", "library"] , "name": ["git_operations"] , "hdrs": ["git_operations.hpp"] , "srcs": ["git_operations.cpp"] , "deps": ["git_ops_types", ["src/buildtool/multithreading", "async_map_consumer"]] , "stage": ["src", "other_tools", "git_operations"] , "private-deps": [ "git_repo_remote" , ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "file_system_manager"] ] } , "git_repo_remote": { "type": ["@", "rules", "CC", "library"] , "name": ["git_repo_remote"] , "hdrs": ["git_repo_remote.hpp"] , "srcs": ["git_repo_remote.cpp"] , "deps": [ ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "git_utils"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "git_operations"] , "private-deps": [ "git_config_settings" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["", "libgit2"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/system", "system_command"] , ["src/utils/cpp", "tmp_dir"] ] } , "git_config_settings": { "type": ["@", "rules", "CC", "library"] , "name": ["git_config_settings"] , "hdrs": ["git_config_settings.hpp"] , "srcs": ["git_config_settings.cpp"] , "stage": ["src", "other_tools", "git_operations"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["", "libgit2"] , ["src/other_tools/utils", "curl_url_handle"] ] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_config_settings.cpp000066400000000000000000000520401516554100600325320ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/git_operations/git_config_settings.hpp" #include #include #include #include #include "fmt/core.h" #include "gsl/gsl" #include "src/other_tools/utils/curl_url_handle.hpp" extern "C" { #include } namespace { void config_iter_closer(gsl::owner iter) { git_config_iterator_free(iter); } // callback to enable SSL certificate check for remote fetch const auto kCertificateCheck = [](git_cert* /*cert*/, int /*valid*/, const char* /*host*/, void* /*payload*/) -> int { return 1; }; // callback to remote fetch without an SSL certificate check const auto kCertificatePassthrough = [](git_cert* /*cert*/, int /*valid*/, const char* /*host*/, void* /*payload*/) -> int { return 0; }; /// \brief Custom comparison of matching degrees. Return true if left argument's /// degree of matching is better that the right argument's. When both are /// equally good matches, return true to make the latest entry win. struct ConfigKeyMatchCompare { [[nodiscard]] auto operator()(ConfigKeyMatchDegree const& left, ConfigKeyMatchDegree const& right) const -> bool { if (left.host_len != right.host_len) { return left.host_len > right.host_len; } if (left.path_len != right.path_len) { return left.path_len > right.path_len; } if (left.user_matched != right.user_matched) { return left.user_matched; } return true; } }; /// \brief Tries to parse the given proxy string as an URL using the libcurl API /// in a more permissive way, mirroring what git and curl internally do, then /// returns the reconstructed URL if parsing succeeded, or a nullopt ProxyInfo. /// Returns nullopt on unexpected errors. [[nodiscard]] auto GetProxyAsPermissiveUrl( std::string const& proxy_url) noexcept -> std::optional { // parse proxy string with permissive options: // use_non_support_scheme allows for non-standard schemes to be parsed; // use_guess_scheme tries to figure out the scheme from the hostname if none // is provided and defaults to http if it fails; auto parsed_url = CurlURLHandle::CreatePermissive(proxy_url, true /*use_guess_scheme*/, false /*use_default_scheme*/, true /*use_non_support_scheme*/); if (not parsed_url) { // exception encountered return std::nullopt; } if (not *parsed_url) { // failure to parse return ProxyInfo{std::nullopt}; } // recombine the parsed url without changing it any further return ProxyInfo{parsed_url.value()->GetURL()}; } } // namespace auto GitConfigSettings::GetSSLCallback(std::shared_ptr const& cfg, std::string const& url, anon_logger_ptr const& logger) -> std::optional { try { // check SSL verification settings, from most to least specific std::optional check_cert{std::nullopt}; int tmp{}; // check if GIT_SSL_NO_VERIFY envariable is set (value is // irrelevant) const char* ssl_no_verify_var{std::getenv("GIT_SSL_NO_VERIFY")}; if (ssl_no_verify_var != nullptr) { check_cert = false; } else { if (cfg != nullptr) { // check all the url-specific gitconfig entries; if any key // url matches, use the respective gitconfig entry value auto parsed_url = CurlURLHandle::Create(url); if (not parsed_url) { // unexpected error occurred (*logger)( "While getting SSL callback:\nfailed to parse remote " "URL", true /*fatal*/); return std::nullopt; } if (*parsed_url) { // iterate over config entries of type // "http..sslVerify" git_config_iterator* iter_ptr{nullptr}; if (git_config_iterator_glob_new( &iter_ptr, cfg.get(), R"(http\..*\.sslverify)") == 0) { // wrap iterator auto iter = std::unique_ptr( iter_ptr, config_iter_closer); // set config key parsing offsets const std::string::difference_type start_offset{ 5}; // len("http.") const std::string::difference_type end_offset{ 10}; // len(".sslverify") // define ordered container storing matches std::map matches{}; // iterate through config keys git_config_entry* entry{nullptr}; while (git_config_next(&entry, iter.get()) == 0) { // get the url part of the config key std::string entry_name{entry->name}; auto entry_url = std::string(entry_name.begin() + start_offset, entry_name.end() - end_offset); // get match degree auto match = parsed_url.value()->MatchConfigKey(entry_url); if (not match) { // unexpected behavior (*logger)( "While getting SSL callback:\nmatching " "config key failed", true /*fatal*/); return std::nullopt; } // store in ordered list only if a match // occurred if (match->matched) { matches.emplace(*match, std::string(entry->value)); } } // if at least one match occurred, use the best one if (not matches.empty()) { if (git_config_parse_bool( &tmp, matches.begin()->second.c_str()) == 0) { check_cert = tmp == 1; } } } } if (not check_cert) { // check the generic gitconfig entry; ignore errors if (git_config_get_bool( &tmp, cfg.get(), R"(http.sslverify)") == 0) { check_cert = tmp == 1; } } } } // set callback: passthrough only if check_cert is false return (check_cert and not *check_cert) ? kCertificatePassthrough : kCertificateCheck; } catch (std::exception const& ex) { (*logger)( fmt::format("Getting SSL callback failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } } auto GitConfigSettings::GetProxySettings(std::shared_ptr const& cfg, std::string const& url, anon_logger_ptr const& logger) -> std::optional { try { // perform proxy checks as git does git_buf tmp_buf{}; // temp buffer if (cfg != nullptr) { // parse given url auto parsed_url = CurlURLHandle::Create(url); if (not parsed_url) { // unexpected error occurred (*logger)( "While getting proxy settings:\nfailed to parse remote URL", true /*fatal*/); return std::nullopt; } if (parsed_url.value()) { // check for no_proxy envariable if (const char* envar = std::getenv("no_proxy")) { // check if there is a pattern match auto is_matched = parsed_url.value()->NoproxyStringMatches(envar); if (not is_matched) { // unexpected error occurred (*logger)( "While getting proxy settings:\nmatching no_proxy " "envariable patterns failed", true /*fatal*/); return std::nullopt; } if (is_matched == true) { return ProxyInfo{std::nullopt}; } } // check for NO_PROXY envariable if (const char* envar = std::getenv("NO_PROXY")) { // check if there is a pattern match auto is_matched = parsed_url.value()->NoproxyStringMatches(envar); if (not is_matched) { // unexpected error occurred (*logger)( "While getting proxy settings:\nmatching NO_PROXY " "envariable patterns failed", true /*fatal*/); return std::nullopt; } if (is_matched == true) { return ProxyInfo{std::nullopt}; } } // iterate over config entries of type // "http..proxy" git_config_iterator* iter_ptr{nullptr}; if (git_config_iterator_glob_new( &iter_ptr, cfg.get(), R"(http\..*\.proxy)") == 0) { // wrap iterator auto iter = std::unique_ptr( iter_ptr, config_iter_closer); // set config key parsing offsets const std::string::difference_type start_offset{ 5}; // len("http.") const std::string::difference_type end_offset{ 6}; // len(".proxy") // define ordered container storing matches of git // config keys std::map matches{}; // iterate through config keys git_config_entry* entry{nullptr}; while (git_config_next(&entry, iter.get()) == 0) { // get the url part of the config key std::string entry_name{entry->name}; auto entry_url = std::string(entry_name.begin() + start_offset, entry_name.end() - end_offset); // get match degree auto match = parsed_url.value()->MatchConfigKey(entry_url); if (not match) { // unexpected behavior (*logger)( "While getting proxy settings:\nmatching " "config key failed", true /*fatal*/); return std::nullopt; } // store in ordered list only if a match occurred if (match->matched) { matches.emplace(*match, std::string(entry->value)); } } // look for any empty proxy value; if found, proxy is // disabled for (auto const& elem : matches) { if (git_config_parse_path(&tmp_buf, elem.second.c_str()) == 0) { if (tmp_buf.size == 0) { // cleanup memory git_buf_dispose(&tmp_buf); return ProxyInfo{std::nullopt}; } // cleanup memory git_buf_dispose(&tmp_buf); } } // no_proxy checks are done, so look for actual proxy info; // first, check the top "http..proxy" match if (not matches.empty()) { if (git_config_parse_path( &tmp_buf, matches.begin()->second.c_str()) == 0) { auto tmp_str = std::string(tmp_buf.ptr); // cleanup memory git_buf_dispose(&tmp_buf); // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(tmp_str); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy " "settings:\npermissive parsing of " "remote-specific proxy URL failed", true /*fatal*/); return std::nullopt; } return proxy_info; } } // check the generic "http.proxy" gitconfig entry; // ignore errors if (git_config_get_string_buf( &tmp_buf, cfg.get(), R"(http.proxy)") == 0) { if (tmp_buf.size > 0) { auto tmp_str = std::string(tmp_buf.ptr); // cleanup memory git_buf_dispose(&tmp_buf); // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(tmp_str); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of http.proxy URL failed", true /*fatal*/); return std::nullopt; } return proxy_info; } // cleanup memory git_buf_dispose(&tmp_buf); } // check proxy envariables depending on the scheme auto url_scheme = parsed_url.value()->GetScheme(); if (not url_scheme) { // unexpected behavior (*logger)( "While getting proxy settings:\nretrieving scheme " "from parsed URL failed", true /*fatal*/); return std::nullopt; } if (url_scheme.value() == "https") { // check https_proxy envariable if (const char* envar = std::getenv("https_proxy")) { // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(envar); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of https_proxy envariable failed", true /*fatal*/); return std::nullopt; } return proxy_info; } // check HTTPS_PROXY envariable if (const char* envar = std::getenv("HTTPS_PROXY")) { // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(envar); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of HTTPS_PROXY envariable failed", true /*fatal*/); return std::nullopt; } return proxy_info; } } else if (url_scheme.value() == "http") { // check http_proxy envariable if (const char* envar = std::getenv("http_proxy")) { // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(envar); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of http_proxy envariable failed", true /*fatal*/); return std::nullopt; } return proxy_info; } } // check all_proxy envariable if (const char* envar = std::getenv("all_proxy")) { // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(envar); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of all_proxy envariable failed", true /*fatal*/); return std::nullopt; } return proxy_info; } // check ALL_PROXY envariable if (const char* envar = std::getenv("ALL_PROXY")) { // get proxy url in standard form auto proxy_info = GetProxyAsPermissiveUrl(envar); if (not proxy_info) { // unexpected behavior (*logger)( "While getting proxy settings:\npermissive " "parsing of ALL_PROXY envariable failed", true /*fatal*/); return std::nullopt; } return proxy_info; } } } } return ProxyInfo{std::nullopt}; // default to disabling proxy } catch (std::exception const& ex) { (*logger)( fmt::format("Getting proxy settings failed with:\n{}", ex.what()), true /*fatal*/); return std::nullopt; } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_config_settings.hpp000066400000000000000000000047531516554100600325470ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_CONFIG_SETTINGS_HPP #define INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_CONFIG_SETTINGS_HPP #include #include #include #include extern "C" { struct git_cert; struct git_config; using git_transport_certificate_check_cb = auto (*)(git_cert*, int, const char*, void*) -> int; } // Internal type used for logging with AsyncMaps using anon_logger_t = std::function; using anon_logger_ptr = std::shared_ptr; /// \brief Contains the proxy URL if proxy is set, or nullopt if proxy unset. using ProxyInfo = std::optional; namespace GitConfigSettings { /// \brief Get a custom SSL certificate check callback to honor the existing /// Git configuration of a repository trying to connect to a remote. /// A null config snapshot reference will simply be ignored. /// Returns nullopt if errors. [[nodiscard]] auto GetSSLCallback(std::shared_ptr const& cfg, std::string const& url, anon_logger_ptr const& logger) -> std::optional; /// \brief Get the remote proxy settings from envariables and the given git /// config snapshot. Performs same checks and honors same settings as git. /// Returns the proxy state and information, or nullopt if errors. [[nodiscard]] auto GetProxySettings(std::shared_ptr const& cfg, std::string const& url, anon_logger_ptr const& logger) -> std::optional; } // namespace GitConfigSettings #endif // INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_CONFIG_SETTINGS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_operations.cpp000066400000000000000000000220411516554100600315260ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/git_operations/git_operations.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" auto CriticalGitOps::GitInitialCommit(GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue { #ifndef NDEBUG // Check required fields have been set if (not crit_op_params.message) { (*logger)("missing message for operation creating commit", true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } if (not crit_op_params.source_path) { (*logger)("missing source_path for operation creating commit", true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } #endif // Create and open a GitRepoRemote at given target location auto git_repo = GitRepoRemote::InitAndOpen(crit_op_params.target_path, /*is_bare=*/false); if (git_repo == std::nullopt or git_repo->GetGitCAS() == nullptr) { (*logger)(fmt::format("could not initialize git repository {}", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While doing initial commit Git op:\n{}", msg), fatal); }); // Commit all at the target directory auto commit_hash = git_repo->CommitDirectory(crit_op_params.source_path.value(), crit_op_params.message.value(), wrapped_logger); if (commit_hash == std::nullopt) { return {.git_cas = nullptr, .result = std::nullopt}; } // success return {.git_cas = git_repo->GetGitCAS(), .result = std::move(commit_hash)}; } auto CriticalGitOps::GitEnsureInit(GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue { // Make sure folder we want to access exists if (not FileSystemManager::CreateDirectory(crit_op_params.target_path)) { (*logger)(fmt::format("target directory {} could not be created", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // Create and open a GitRepo at given target location auto git_repo = GitRepoRemote::InitAndOpen( crit_op_params.target_path, /*is_bare=*/crit_op_params.init_bare.value()); if (git_repo == std::nullopt or git_repo->GetGitCAS() == nullptr) { (*logger)( fmt::format("could not initialize {} git repository {}", crit_op_params.init_bare.value() ? "bare" : "non-bare", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // success return {.git_cas = git_repo->GetGitCAS(), .result = ""}; } auto CriticalGitOps::GitKeepTag(GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue { #ifndef NDEBUG // Check required fields have been set if (not crit_op_params.message) { (*logger)("missing message for operation tagging a commit", true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } #endif // Make sure folder we want to access exists if (not FileSystemManager::Exists(crit_op_params.target_path)) { (*logger)(fmt::format("target directory {} does not exist!", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // Open a GitRepo at given location auto git_repo = GitRepoRemote::Open(crit_op_params.target_path); if (git_repo == std::nullopt) { (*logger)(fmt::format("could not open git repository {}", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While doing keep tag Git op:\n{}", msg), fatal); }); // Create tag of given commit auto tag_result = git_repo->KeepTag(crit_op_params.git_hash, crit_op_params.message.value(), wrapped_logger); if (not tag_result) { return {.git_cas = nullptr, .result = std::nullopt}; } // success return {.git_cas = git_repo->GetGitCAS(), .result = std::move(tag_result)}; } auto CriticalGitOps::GitGetHeadId(GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue { // Make sure folder we want to access exists if (not FileSystemManager::Exists(crit_op_params.target_path)) { (*logger)(fmt::format("target directory {} does not exist!", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // Open a GitRepo at given location auto git_repo = GitRepoRemote::Open(crit_op_params.target_path); if (git_repo == std::nullopt) { (*logger)(fmt::format("could not open git repository {}", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While doing get HEAD id Git op:\n{}", msg), fatal); }); // Get head commit auto head_commit = git_repo->GetHeadCommit(wrapped_logger); if (head_commit == std::nullopt) { return {.git_cas = nullptr, .result = std::nullopt}; } // success return {.git_cas = git_repo->GetGitCAS(), .result = std::move(head_commit)}; } auto CriticalGitOps::GitKeepTree(GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue { #ifndef NDEBUG // Check required fields have been set if (not crit_op_params.message) { (*logger)("missing message for operation keeping a tree comitted", true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } #endif // Make sure folder we want to access exists if (not FileSystemManager::Exists(crit_op_params.target_path)) { (*logger)(fmt::format("target directory {} does not exist!", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // Open a GitRepo at given location auto git_repo = GitRepoRemote::Open(crit_op_params.target_path); if (git_repo == std::nullopt) { (*logger)(fmt::format("could not open git repository {}", crit_op_params.target_path.string()), true /*fatal*/); return {.git_cas = nullptr, .result = std::nullopt}; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While doing keep tree Git op:\n{}", msg), fatal); }); // Create tag for given tree auto tag_result = git_repo->KeepTree(crit_op_params.git_hash, crit_op_params.message.value(), wrapped_logger); if (not tag_result) { return {.git_cas = nullptr, .result = std::nullopt}; } // success return {.git_cas = git_repo->GetGitCAS(), .result = std::move(tag_result)}; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_operations.hpp000066400000000000000000000062611516554100600315410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPERATIONS_HPP #define INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPERATIONS_HPP #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" /// \brief Class defining the critical Git operations, i.e., those which write /// to the underlying Git ODB. /// The target_path is a mandatory argument, as it is used in a file locking /// mechanism, ensuring only one process at a time works on a particular /// repository on the file system. class CriticalGitOps { public: // This operations needs the params: target_path, message // Will perform: "git init && git add . && git commit -m ". // Called to setup first commit in new repository. Assumes folder exists. // It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GitInitialCommit( GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue; // This operation needs the params: target_path // Called to initialize a repository. Creates folder if not there. // It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GitEnsureInit( GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue; // This operation needs the params: target_path, git_hash (commit), message // Called after a git fetch to retain the commit. Assumes folder exists. // It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GitKeepTag( GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue; // This operations needs the params: target_path // Called to retrieve the HEAD commit hash. Assumes folder exists. // It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GitGetHeadId( GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue; // This operation needs the params: target_path, git_hash (tree), message // Called after resolving symlinks in a tree to retain the resolved tree // by tagging it. Assumes folder exists. // It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] static auto GitKeepTree( GitOpParams const& crit_op_params, AsyncMapConsumerLoggerPtr const& logger) -> GitOpValue; }; #endif // INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPERATIONS_HPPjust-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_ops_types.hpp000066400000000000000000000053311516554100600314000ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPS_TYPES_HPP #define INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPS_TYPES_HPP #include #include #include #include #include // std::move #include "src/buildtool/file_system/git_cas.hpp" #include "src/utils/cpp/path.hpp" /// \brief Common parameters for all critical Git operations struct GitOpParams { std::filesystem::path target_path; /*key*/ std::string git_hash; /*key*/ std::optional message{ std::nullopt}; // mandatory for commits and tags std::optional source_path{ std::nullopt}; // mandatory for commits std::optional init_bare{std::nullopt}; // useful for git init GitOpParams( std::filesystem::path const& target_path_, std::string git_hash_, std::optional message_ = std::nullopt, std::optional source_path_ = std::nullopt, std::optional init_bare_ = std::nullopt) : target_path{std::filesystem::absolute(ToNormalPath(target_path_))}, git_hash{std::move(git_hash_)}, message{std::move(message_)}, source_path{std::move(source_path_)}, init_bare{init_bare_} {}; [[nodiscard]] auto operator==(GitOpParams const& other) const noexcept -> bool { // not all fields are keys return target_path == other.target_path and git_hash == other.git_hash; } }; /// \brief Defines the type of Git operation enum class GitOpType : std::uint8_t { DEFAULT_OP, // default value; does nothing INITIAL_COMMIT, ENSURE_INIT, KEEP_TAG, GET_HEAD_ID, KEEP_TREE }; /// \brief Common return value for all critical Git operations struct GitOpValue { // used to continue with non-critical ops on same odb, if needed GitCASPtr git_cas{nullptr}; // stores result of certain operations; always nullopt if failure std::optional result = std::nullopt; }; #endif // INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_OPS_TYPES_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_repo_remote.cpp000066400000000000000000000636631516554100600317020ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/git_operations/git_repo_remote.hpp" #include #include #include #include #include #include #include // std::move #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_config_settings.hpp" #include "src/utils/cpp/tmp_dir.hpp" extern "C" { #include #include } namespace { /// \brief Basic check for libgit2 protocols we support. For all other cases, we /// should default to shell out to git instead. [[nodiscard]] auto IsSupported(std::string const& url) -> bool { // look for explicit schemes if (url.starts_with("git://") or url.starts_with("http://") or url.starts_with("https://") or url.starts_with("file://")) { return true; } // look for url as existing filesystem directory path if (FileSystemManager::IsDirectory(std::filesystem::path(url))) { return true; } return false; // as default } struct FetchIntoODBBackend { git_odb_backend parent; git_odb* target_odb; // the odb where the fetch objects will end up into }; [[nodiscard]] auto fetch_backend_writepack(git_odb_writepack** _writepack, git_odb_backend* _backend, [[maybe_unused]] git_odb* odb, git_indexer_progress_cb progress_cb, void* progress_payload) -> int { Ensures(_backend != nullptr); auto* b = reinterpret_cast(_backend); // NOLINT return git_odb_write_pack( _writepack, b->target_odb, progress_cb, progress_payload); } [[nodiscard]] auto fetch_backend_exists(git_odb_backend* _backend, const git_oid* oid) -> int { Ensures(_backend != nullptr); auto* b = reinterpret_cast(_backend); // NOLINT return git_odb_exists(b->target_odb, oid); } void fetch_backend_free(git_odb_backend* /*_backend*/) {} [[nodiscard]] auto CreateFetchIntoODBParent() -> git_odb_backend { git_odb_backend b{}; b.version = GIT_ODB_BACKEND_VERSION; // only populate the functions needed b.writepack = &fetch_backend_writepack; // needed for fetch b.exists = &fetch_backend_exists; b.free = &fetch_backend_free; return b; } // A backend that can be used to fetch from the remote of another repository. auto const kFetchIntoODBParent = CreateFetchIntoODBParent(); } // namespace auto GitRepoRemote::Open(GitCASPtr git_cas) noexcept -> std::optional { auto repo = GitRepoRemote(std::move(git_cas)); if (not repo.GetGitCAS()) { return std::nullopt; } return repo; } auto GitRepoRemote::Open(std::filesystem::path const& repo_path) noexcept -> std::optional { auto repo = GitRepoRemote(repo_path); if (not repo.GetGitCAS()) { return std::nullopt; } return repo; } GitRepoRemote::GitRepoRemote(GitCASPtr git_cas) noexcept : GitRepo(std::move(git_cas)) {} GitRepoRemote::GitRepoRemote(std::filesystem::path const& repo_path) noexcept : GitRepo(repo_path) {} GitRepoRemote::GitRepoRemote(GitRepo&& other) noexcept : GitRepo(std::move(other)) {} GitRepoRemote::GitRepoRemote(GitRepoRemote&& other) noexcept : GitRepo(std::move(other)) {} auto GitRepoRemote::operator=(GitRepoRemote&& other) noexcept -> GitRepoRemote& { GitRepo::operator=(std::move(other)); return *this; } auto GitRepoRemote::InitAndOpen(std::filesystem::path const& repo_path, bool is_bare) noexcept -> std::optional { auto res = GitRepo::InitAndOpen(repo_path, is_bare); if (res) { return GitRepoRemote(std::move(*res)); } return std::nullopt; } auto GitRepoRemote::GetCommitFromRemote(std::shared_ptr cfg, std::string const& repo_url, std::string const& branch, anon_logger_ptr const& logger) noexcept -> std::optional { try { // only possible for real repository! if (IsRepoFake()) { (*logger)("Cannot update commit using a fake repository!", true /*fatal*/); return std::nullopt; } // create remote git_remote* remote_ptr{nullptr}; if (git_remote_create_anonymous(&remote_ptr, GetGitCAS()->GetRepository(), repo_url.c_str()) != 0) { (*logger)( fmt::format("Creating anonymous remote for git repository {} " "failed with:\n{}", GetGitCAS()->GetPath().string(), GitLastError()), true /*fatal*/); git_remote_free(remote_ptr); return std::nullopt; } auto remote = std::unique_ptr( remote_ptr, remote_closer); // get the canonical url auto canonical_url = std::string(git_remote_url(remote.get())); // get a well-defined config file if (not cfg) { // get config snapshot of current repo (shared with caller) cfg = GetConfigSnapshot(); if (cfg == nullptr) { (*logger)(fmt::format("Retrieving config object in get commit " "from remote failed with:\n{}", GitLastError()), true /*fatal*/); return std::nullopt; } } // connect to remote git_remote_callbacks callbacks{}; git_remote_init_callbacks(&callbacks, GIT_REMOTE_CALLBACKS_VERSION); // set custom SSL verification callback; use canonicalized url auto cert_check = GitConfigSettings::GetSSLCallback(cfg, canonical_url, logger); if (not cert_check) { // error occurred while handling the url return std::nullopt; } callbacks.certificate_check = *cert_check; git_proxy_options proxy_opts{}; git_proxy_options_init(&proxy_opts, GIT_PROXY_OPTIONS_VERSION); // set the proxy information auto proxy_info = GitConfigSettings::GetProxySettings(cfg, canonical_url, logger); if (not proxy_info) { // error occurred while handling the url return std::nullopt; } if (proxy_info.value()) { // found proxy proxy_opts.type = GIT_PROXY_SPECIFIED; proxy_opts.url = proxy_info.value().value().c_str(); } else { // no proxy proxy_opts.type = GIT_PROXY_NONE; } if (git_remote_connect(remote.get(), GIT_DIRECTION_FETCH, &callbacks, &proxy_opts, nullptr) != 0) { (*logger)( fmt::format("Connecting to remote {} for git repository {} " "failed with:\n{}", repo_url, GetGitCAS()->GetPath().string(), GitLastError()), true /*fatal*/); return std::nullopt; } // get the list of refs from remote // NOTE: refs will be owned by remote, so we DON'T have to free it! git_remote_head const** refs = nullptr; std::size_t refs_len = 0; if (git_remote_ls(&refs, &refs_len, remote.get()) != 0) { (*logger)( fmt::format("Refs retrieval from remote {} failed with:\n{}", repo_url, GitLastError()), true /*fatal*/); return std::nullopt; } // figure out what remote branch the local one is tracking for (size_t i = 0; i < refs_len; ++i) { // by treating each read reference string as a path we can easily // check for the branch name // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) std::filesystem::path ref_name_as_path{refs[i]->name}; if (ref_name_as_path.filename() == branch) { // branch found! // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) std::string new_commit_hash{git_oid_tostr_s(&refs[i]->oid)}; return new_commit_hash; } } (*logger)(fmt::format("Could not find branch {} for remote {}", branch, repo_url, GitLastError()), true /*fatal*/); return std::nullopt; } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Get commit from branch {} of remote {} failed with:\n{}", branch, repo_url, ex.what()); return std::nullopt; } } auto GitRepoRemote::FetchFromRemote(std::shared_ptr cfg, std::string const& repo_url, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool { try { // only possible for real repository! if (IsRepoFake()) { (*logger)("Cannot fetch commit using a fake repository!", true /*fatal*/); return false; } // create remote from repo git_remote* remote_ptr{nullptr}; if (git_remote_create_anonymous(&remote_ptr, GetGitCAS()->GetRepository(), repo_url.c_str()) != 0) { (*logger)(fmt::format("Creating remote {} for git repository {} " "failed with:\n{}", repo_url, GetGitCAS()->GetPath().string(), GitLastError()), true /*fatal*/); // cleanup resources git_remote_free(remote_ptr); return false; } // wrap remote object auto remote = std::unique_ptr( remote_ptr, remote_closer); // get the canonical url auto canonical_url = std::string(git_remote_url(remote.get())); // get a well-defined config file if (not cfg) { // get config snapshot of current repo git_config* cfg_ptr{nullptr}; if (git_repository_config_snapshot( &cfg_ptr, GetGitCAS()->GetRepository()) != 0) { (*logger)(fmt::format("Retrieving config object in fetch from " "remote failed with:\n{}", GitLastError()), true /*fatal*/); return false; } // share pointer with caller cfg.reset(cfg_ptr, config_closer); } // define default fetch options git_fetch_options fetch_opts{}; git_fetch_options_init(&fetch_opts, GIT_FETCH_OPTIONS_VERSION); // set the proxy information auto proxy_info = GitConfigSettings::GetProxySettings(cfg, canonical_url, logger); if (not proxy_info) { // error occurred while handling the url return false; } if (proxy_info.value()) { // found proxy fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; fetch_opts.proxy_opts.url = proxy_info.value().value().c_str(); } else { // no proxy fetch_opts.proxy_opts.type = GIT_PROXY_NONE; } // set custom SSL verification callback; use canonicalized url if (not cfg) { // get config snapshot of current repo (shared with caller) cfg = GetConfigSnapshot(); if (cfg == nullptr) { (*logger)(fmt::format("Retrieving config object in fetch from " "remote failed with:\n{}", GitLastError()), true /*fatal*/); return false; } } auto cert_check = GitConfigSettings::GetSSLCallback(cfg, canonical_url, logger); if (not cert_check) { // error occurred while handling the url return false; } fetch_opts.callbacks.certificate_check = *cert_check; // disable update of the FETCH_HEAD pointer fetch_opts.update_fetchhead = 0; // setup fetch refspecs array GitStrArray refspecs_array_obj; if (branch) { // make sure we check for tags as well refspecs_array_obj.AddEntry(fmt::format("+refs/tags/{}", *branch)); refspecs_array_obj.AddEntry(fmt::format("+refs/heads/{}", *branch)); } auto const refspecs_array = refspecs_array_obj.Get(); if (git_remote_fetch( remote.get(), &refspecs_array, &fetch_opts, nullptr) != 0) { (*logger)( fmt::format("Fetching{} in git repository {} failed " "with:\n{}", branch ? fmt::format(" branch {}", *branch) : "", GetGitCAS()->GetPath().string(), GitLastError()), true /*fatal*/); return false; } return true; // success! } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Fetch{} from remote {} failed with:\n{}", branch ? fmt::format(" branch {}", *branch) : "", repo_url, ex.what()); return false; } } auto GitRepoRemote::UpdateCommitViaTmpRepo( StorageConfig const& storage_config, std::string const& repo_url, std::string const& branch, std::vector const& inherit_env, std::string const& git_bin, std::vector const& launcher, anon_logger_ptr const& logger) const noexcept -> std::optional { try { auto tmp_dir = storage_config.CreateTypedTmpDir("update"); if (not tmp_dir) { (*logger)("Failed to create temp dir for running 'git ls-remote'", /*fatal=*/true); return std::nullopt; } auto const& tmp_path = tmp_dir->GetPath(); // check for internally supported protocols if (IsSupported(repo_url)) { // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Commit update called on a real repository"); } // create the temporary real repository auto tmp_repo = GitRepoRemote::InitAndOpen(tmp_path / "git", /*is_bare=*/true); if (tmp_repo == std::nullopt) { return std::nullopt; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format( "While doing commit update via tmp repo:\n{}", msg), fatal); }); // get the config of the correct target repo auto cfg = GetConfigSnapshot(); if (cfg == nullptr) { (*logger)( fmt::format("Retrieving config object in update commit " "via tmp repo failed with:\n{}", GitLastError()), true /*fatal*/); return std::nullopt; } return tmp_repo->GetCommitFromRemote( cfg, repo_url, branch, wrapped_logger); } // default to shelling out to git for non-explicitly supported protocols auto cmdline = launcher; cmdline.insert(cmdline.end(), {git_bin, "ls-remote", repo_url, branch}); Logger::Log( LogLevel::Debug, "Git commit update for remote {} must shell out. Running:\n{}", repo_url, nlohmann::json(cmdline).dump()); std::map env{}; for (auto const& k : inherit_env) { const char* v = std::getenv(k.c_str()); if (v != nullptr) { env[k] = std::string(v); } } // set up the system command SystemCommand system{repo_url}; auto const exit_code = system.Execute( cmdline, env, GetGitCAS()->GetPath(), // which path is not actually relevant tmp_path); if (not exit_code) { (*logger)(fmt::format("exec() on command failed."), /*fatal=*/true); return std::nullopt; } // output file can be read anyway std::string out_str{}; auto cmd_out = FileSystemManager::ReadFile(tmp_path / "stdout"); if (cmd_out) { out_str = *cmd_out; } // check for command failure if (*exit_code != 0) { std::string err_str{}; auto cmd_err = FileSystemManager::ReadFile(tmp_path / "stderr"); if (cmd_err) { err_str = *cmd_err; } std::string output{}; if (not out_str.empty() or not err_str.empty()) { output = fmt::format(" with output:\n{}{}", out_str, err_str); } (*logger)(fmt::format("List remote commits command {} failed{}", nlohmann::json(cmdline).dump(), output), /*fatal=*/true); return std::nullopt; } // report failure to read generated output file or the file is empty if (out_str.empty()) { (*logger)(fmt::format("List remote commits command {} failed to " "produce an output", nlohmann::json(cmdline).dump()), /*fatal=*/true); return std::nullopt; } // parse the output: it should contain two tab-separated columns, // with the commit being the first entry auto str_len = out_str.find('\t'); if (str_len == std::string::npos) { (*logger)(fmt::format("List remote commits command {} produced " "malformed output:\n{}", nlohmann::json(cmdline).dump(), out_str), /*fatal=*/true); return std::nullopt; } // success! return out_str.substr(0, str_len); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Update commit from branch {} of remote {} via tmp dir " "failed with:\n{}", branch, repo_url, ex.what()); return std::nullopt; } } auto GitRepoRemote::FetchViaTmpRepo(StorageConfig const& storage_config, std::string const& repo_url, std::optional const& branch, std::vector const& inherit_env, std::string const& git_bin, std::vector const& launcher, anon_logger_ptr const& logger) noexcept -> bool { try { auto tmp_dir = storage_config.CreateTypedTmpDir("fetch"); if (not tmp_dir) { (*logger)("Failed to create temp dir for running 'git fetch'", /*fatal=*/true); return false; } auto const& tmp_path = tmp_dir->GetPath(); // check for internally supported protocols if (IsSupported(repo_url)) { Logger::Log(LogLevel::Debug, "Try fetch from URL {}", repo_url); // preferably with a "fake" repository! if (not IsRepoFake()) { Logger::Log(LogLevel::Debug, "Branch fetch called on a real repository"); } // create the temporary real repository // it can be bare, as the refspecs for this fetch will be given // explicitly. auto tmp_repo = GitRepoRemote::InitAndOpen(tmp_path / "git", /*is_bare=*/true); if (tmp_repo == std::nullopt) { return false; } // add backend, with max priority FetchIntoODBBackend b{.parent = kFetchIntoODBParent, .target_odb = GetGitCAS()->GetODB()}; if (git_odb_add_backend( tmp_repo->GetGitCAS()->GetODB(), // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) reinterpret_cast(&b), std::numeric_limits::max()) == 0) { // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While doing branch fetch via " "tmp repo:\n{}", msg), fatal); }); // get the config of the correct target repo auto cfg = GetConfigSnapshot(); if (cfg == nullptr) { (*logger)( fmt::format("Retrieving config object in fetch via " "tmp repo failed with:\n{}", GitLastError()), true /*fatal*/); return false; } return tmp_repo->FetchFromRemote( cfg, repo_url, branch, wrapped_logger); } return false; } // default to shelling out to git for non-explicitly supported protocols auto cmdline = launcher; // Note: Because we fetch with URL, not a known remote, no refs are // being updated by default, so git has no reason to get a lock // file. This however does not imply automatically that fetches // might not internally wait for each other through other means. cmdline.insert(cmdline.end(), {git_bin, "fetch", "--no-auto-gc", "--no-write-fetch-head", repo_url}); if (branch) { cmdline.push_back(*branch); } Logger::Log(LogLevel::Debug, "Git fetch for remote {} must shell out. Running:\n{}", repo_url, nlohmann::json(cmdline).dump()); std::map env{}; for (auto const& k : inherit_env) { const char* v = std::getenv(k.c_str()); if (v != nullptr) { env[k] = std::string(v); } } // run command SystemCommand system{repo_url}; auto const exit_code = system.Execute(cmdline, env, GetGitCAS()->GetPath(), tmp_path); if (not exit_code) { (*logger)(fmt::format("exec() on command failed."), /*fatal=*/true); return false; } if (*exit_code != 0) { std::string out_str{}; std::string err_str{}; auto cmd_out = FileSystemManager::ReadFile(tmp_path / "stdout"); auto cmd_err = FileSystemManager::ReadFile(tmp_path / "stderr"); if (cmd_out) { out_str = *cmd_out; } if (cmd_err) { err_str = *cmd_err; } std::string output{}; if (not out_str.empty() or not err_str.empty()) { output = fmt::format(" with output:\n{}{}", out_str, err_str); } (*logger)(fmt::format("Fetch command {} failed{}", nlohmann::json(cmdline).dump(), output), /*fatal=*/true); return false; } return true; // fetch successful } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Fetch{} from remote {} via tmp dir failed with:\n{}", branch ? fmt::format(" branch {}", *branch) : "", repo_url, ex.what()); return false; } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/git_operations/git_repo_remote.hpp000066400000000000000000000133551516554100600317000ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_REPO_REMOTE_HPP #define INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_REPO_REMOTE_HPP #include #include #include #include #include #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/storage/config.hpp" extern "C" { struct git_config; } /// \brief Extension to a Git repository, allowing remote Git operations. class GitRepoRemote : public GitRepo { public: GitRepoRemote() = delete; // no default ctor ~GitRepoRemote() noexcept = default; // allow only move, no copy GitRepoRemote(GitRepoRemote const&) = delete; GitRepoRemote(GitRepoRemote&&) noexcept; auto operator=(GitRepoRemote const&) = delete; auto operator=(GitRepoRemote&& other) noexcept -> GitRepoRemote&; /// \brief Factory to wrap existing open CAS in a "fake" repository. [[nodiscard]] static auto Open(GitCASPtr git_cas) noexcept -> std::optional; /// \brief Factory to open existing real repository at given location. [[nodiscard]] static auto Open( std::filesystem::path const& repo_path) noexcept -> std::optional; /// \brief Factory to initialize and open new real repository at location. /// Returns nullopt if repository init fails even after repeated tries. [[nodiscard]] static auto InitAndOpen( std::filesystem::path const& repo_path, bool is_bare) noexcept -> std::optional; /// \brief Retrieve commit hash from remote branch given its name. /// Only possible with real repository and thus non-thread-safe. /// If non-null, use given config snapshot to interact with config entries; /// otherwise, use a snapshot from the current repo and share pointer to it. /// Returns the retrieved commit hash, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto GetCommitFromRemote( std::shared_ptr cfg, std::string const& repo_url, std::string const& branch, anon_logger_ptr const& logger) noexcept -> std::optional; /// \brief Fetch from given remote. It can either fetch a given named /// branch, or it can fetch with base refspecs. /// Only possible with real repository and thus non-thread-safe. /// If non-null, use given config snapshot to interact with config entries; /// otherwise, use a snapshot from the current repo and share pointer to it. /// Returns a success flag. It guarantees the logger is called /// exactly once with fatal if failure. [[nodiscard]] auto FetchFromRemote(std::shared_ptr cfg, std::string const& repo_url, std::optional const& branch, anon_logger_ptr const& logger) noexcept -> bool; /// \brief Get commit from given branch on the remote. If URL is SSH, shells /// out to system git to perform an ls-remote call, ensuring correct /// handling of the remote connection settings (in particular proxy and /// SSH). For non-SSH URLs, the branch commit is retrieved asynchronously /// using libgit2. /// Returns the commit hash, as a string, or nullopt if failure. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto UpdateCommitViaTmpRepo( StorageConfig const& storage_config, std::string const& repo_url, std::string const& branch, std::vector const& inherit_env, std::string const& git_bin, std::vector const& launcher, anon_logger_ptr const& logger) const noexcept -> std::optional; /// \brief Fetch from a remote. If URL is SSH, shells out to system git to /// retrieve packs in a safe manner, with the only side-effect being that /// there can be some redundancy in the fetched packs. /// For non-SSH URLs an asynchronous fetch is performed using libgit2. /// Uses either a given branch, or fetches all (with base refspecs). /// Returns a success flag. /// It guarantees the logger is called exactly once with fatal if failure. [[nodiscard]] auto FetchViaTmpRepo( StorageConfig const& storage_config, std::string const& repo_url, std::optional const& branch, std::vector const& inherit_env, std::string const& git_bin, std::vector const& launcher, anon_logger_ptr const& logger) noexcept -> bool; private: /// \brief Open "fake" repository wrapper for existing CAS. explicit GitRepoRemote(GitCASPtr git_cas) noexcept; /// \brief Open real repository at given location. explicit GitRepoRemote(std::filesystem::path const& repo_path) noexcept; /// \brief Construct from inherited class. explicit GitRepoRemote(GitRepo&&) noexcept; }; #endif // INCLUDED_SRC_OTHER_TOOLS_GIT_OPERATIONS_GIT_REPO_REMOTE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/000077500000000000000000000000001516554100600244325ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/TARGETS000066400000000000000000000304471516554100600254760ustar00rootroot00000000000000{ "just-mr": { "type": ["@", "rules", "CC", "binary"] , "arguments_config": ["FINAL_LDFLAGS"] , "name": ["just-mr"] , "srcs": ["main.cpp"] , "private-deps": [ "cli" , "exit_codes" , "fetch" , "launch" , "mirrors" , "rc" , "setup" , "setup_utils" , "update" , "utils" , ["@", "cli11", "", "cli11"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "clidefaults"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_context"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "version"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "garbage_collector"] , ["src/buildtool/storage", "repository_garbage_collector"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-ldflags": { "type": "++" , "$1": [ ["-Wl,-z,stack-size=8388608"] , {"type": "var", "name": "FINAL_LDFLAGS", "default": []} ] } } , "utils": { "type": ["@", "rules", "CC", "library"] , "name": ["utils"] , "hdrs": ["utils.hpp"] , "srcs": ["utils.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "exit_codes": { "type": ["@", "rules", "CC", "library"] , "name": ["exit_codes"] , "hdrs": ["exit_codes.hpp"] , "stage": ["src", "other_tools", "just_mr"] } , "cli": { "type": ["@", "rules", "CC", "library"] , "name": ["cli"] , "hdrs": ["cli.hpp"] , "deps": [ "mirrors" , "utils" , ["@", "cli11", "", "cli11"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "clidefaults"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/logging", "log_level"] ] , "stage": ["src", "other_tools", "just_mr"] } , "setup_utils": { "type": ["@", "rules", "CC", "library"] , "name": ["setup_utils"] , "hdrs": ["setup_utils.hpp"] , "srcs": ["setup_utils.cpp"] , "deps": [ "cli" , ["src/buildtool/auth", "auth"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/serve_api/remote", "config"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ "exit_codes" , ["@", "json", "", "json"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "precomputed_root"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/other_tools/utils", "parse_precomputed_root"] , ["src/utils/cpp", "expected"] ] } , "fetch": { "type": ["@", "rules", "CC", "library"] , "name": ["fetch"] , "hdrs": ["fetch.hpp"] , "srcs": ["fetch.cpp"] , "deps": [ "cli" , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ "exit_codes" , "setup_utils" , "utils" , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/execution_api/remote", "bazel_api"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "garbage_collector"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/ops_maps", "archive_fetch_map"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/other_tools/utils", "parse_archive"] , ["src/other_tools/utils", "parse_git_tree"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "file_locking"] ] } , "update": { "type": ["@", "rules", "CC", "library"] , "name": ["update"] , "hdrs": ["update.hpp"] , "srcs": ["update.cpp"] , "deps": [ "cli" , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ "exit_codes" , "utils" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/ops_maps", "git_update_map"] , ["src/utils/cpp", "tmp_dir"] ] } , "setup": { "type": ["@", "rules", "CC", "library"] , "name": ["setup"] , "hdrs": ["setup.hpp"] , "srcs": ["setup.cpp"] , "deps": [ "cli" , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ "setup_utils" , "utils" , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/common/remote", "remote_common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_api/local", "local_api"] , ["src/buildtool/execution_api/remote", "bazel_api"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system/symlinks", "resolve_symlinks_map"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "garbage_collector"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/other_tools/repo_map", "repos_to_setup_map"] , ["src/other_tools/root_maps", "commit_git_map"] , ["src/other_tools/root_maps", "content_git_map"] , ["src/other_tools/root_maps", "distdir_git_map"] , ["src/other_tools/root_maps", "foreign_file_git_map"] , ["src/other_tools/root_maps", "fpath_git_map"] , ["src/other_tools/root_maps", "tree_id_git_map"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "file_locking"] ] } , "launch": { "type": ["@", "rules", "CC", "library"] , "name": ["launch"] , "hdrs": ["launch.hpp"] , "srcs": ["launch.cpp"] , "deps": [ "cli" , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ "exit_codes" , "setup" , "setup_utils" , "utils" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "clidefaults"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/execution_api/common", "ids"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "garbage_collector"] , ["src/buildtool/storage", "repository_garbage_collector"] , ["src/utils/cpp", "file_locking"] , ["src/utils/cpp", "path"] ] } , "mirrors": { "type": ["@", "rules", "CC", "library"] , "name": ["mirrors"] , "hdrs": ["mirrors.hpp"] , "deps": [ ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/other_tools/utils", "curl_url_handle"] ] , "stage": ["src", "other_tools", "just_mr"] } , "rc": { "type": ["@", "rules", "CC", "library"] , "name": ["rc"] , "hdrs": ["rc.hpp"] , "srcs": ["rc.cpp"] , "deps": ["cli", ["@", "gsl", "", "gsl"]] , "private-deps": [ "exit_codes" , "rc_merge" , "utils" , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/common", "clidefaults"] , ["src/buildtool/common", "location"] , ["src/buildtool/common", "retry_cli"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "other_tools", "just_mr"] } , "rc_merge": { "type": ["@", "rules", "CC", "library"] , "name": ["rcmerge"] , "hdrs": ["rc_merge.hpp"] , "srcs": ["rc_merge.cpp"] , "stage": ["src", "other_tools", "just_mr"] , "deps": [["src/buildtool/build_engine/expression", "expression"]] , "private-deps": [["src/buildtool/build_engine/expression", "expression_ptr_interface"]] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/cli.hpp000066400000000000000000000364371516554100600257270ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_CLI_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_CLI_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include "CLI/CLI.hpp" #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/utils.hpp" /// \brief Arguments common to all just-mr subcommands struct MultiRepoCommonArguments { std::optional repository_config{std::nullopt}; std::optional absent_repository_file{std::nullopt}; std::optional checkout_locations_file{std::nullopt}; std::vector explicit_distdirs; LocalPathsPtr just_mr_paths = std::make_shared(); MirrorsPtr alternative_mirrors = std::make_shared(); std::optional> local_launcher{std::nullopt}; CAInfoPtr ca_info = std::make_shared(); std::optional just_path{std::nullopt}; std::optional main{std::nullopt}; std::optional rc_path{std::nullopt}; std::optional git_path{std::nullopt}; std::optional dump_rc{std::nullopt}; bool norc{false}; std::size_t jobs{std::max(1U, std::thread::hardware_concurrency())}; std::vector defines; std::optional remote_execution_address; std::optional remote_instance_name; bool compatible{false}; std::optional remote_serve_address; bool fetch_absent{false}; }; struct MultiRepoLogArguments { std::vector log_files; std::optional log_limit; std::optional restrict_stderr_log_limit; bool plain_log{false}; bool log_append{false}; }; struct MultiRepoSetupArguments { std::optional sub_main{std::nullopt}; bool sub_all{false}; }; struct MultiRepoFetchArguments { std::optional fetch_dir{std::nullopt}; bool backup_to_remote{false}; }; struct MultiRepoUpdateArguments { std::vector repos_to_update; }; struct MultiRepoGcArguments { bool drop_only{false}; }; // Arguments for invocation logging; set only via rc files struct InvocationLogArguments { std::optional directory{std::nullopt}; std::optional invocation_msg{std::nullopt}; std::vector context_vars{}; std::optional project_id{std::nullopt}; std::optional metadata{std::nullopt}; std::optional graph_file{std::nullopt}; std::optional graph_file_plain{std::nullopt}; std::optional dump_artifacts_to_build{std::nullopt}; std::optional dump_artifacts{std::nullopt}; std::optional profile{std::nullopt}; }; struct MultiRepoJustSubCmdsArguments { std::optional subcmd_name{std::nullopt}; std::vector additional_just_args; std::unordered_map> just_args; std::optional config; std::optional endpoint_configuration; }; // corresponding to the similarly-named arguments in 'just' struct MultiRepoRemoteAuthArguments { std::optional tls_ca_cert{std::nullopt}; std::optional tls_client_cert{std::nullopt}; std::optional tls_client_key{std::nullopt}; }; struct ForwardOnlyArguments { std::vector remote_execution_properties; }; enum class SubCommand : std::uint8_t { kUnknown, kMRVersion, kFetch, kUpdate, kSetup, kSetupEnv, kJustDo, kJustSubCmd, kGcRepo }; struct CommandLineArguments { SubCommand cmd{SubCommand::kUnknown}; MultiRepoCommonArguments common; RetryArguments retry; MultiRepoLogArguments log; MultiRepoSetupArguments setup; MultiRepoFetchArguments fetch; MultiRepoUpdateArguments update; MultiRepoGcArguments gc; MultiRepoJustSubCmdsArguments just_cmd; MultiRepoRemoteAuthArguments auth; ForwardOnlyArguments launch_fwd; InvocationLogArguments invocation_log; }; static inline void SetupMultiRepoCommonArguments( gsl::not_null const& app, gsl::not_null const& clargs) { // repository config is mandatory app->add_option_function( "-C, --repository-config", [clargs](auto const& repository_config_raw) { clargs->repository_config = std::filesystem::weakly_canonical(repository_config_raw); }, "Repository-description file to use.") ->type_name("FILE"); app->add_option_function( "--absent", [clargs](auto const& file_raw) { clargs->absent_repository_file = std::filesystem::weakly_canonical(file_raw); }, "File specifying the repositories to consider absent (overrides the " "pragma in the config file).") ->type_name("FILE"); app->add_option_function( "--local-build-root", [clargs](auto const& local_build_root_raw) { clargs->just_mr_paths->root = std::filesystem::weakly_canonical(local_build_root_raw); }, "Root for CAS, repository space, etc.") ->type_name("PATH"); app->add_option_function( "--checkout-locations", [clargs](auto const& checkout_locations_raw) { clargs->checkout_locations_file = std::filesystem::weakly_canonical(checkout_locations_raw); }, "Specification file for checkout locations.") ->type_name("CHECKOUT_LOCATIONS"); app->add_option_function( "-L, --local-launcher", [clargs](auto const& launcher_raw) { clargs->local_launcher = nlohmann::json::parse(launcher_raw) .template get>(); }, "JSON array with the list of strings representing the launcher to " "prepend actions' commands before being executed locally.") ->type_name("JSON"); app->add_option_function( "--distdir", [clargs](auto const& distdir_raw) { clargs->explicit_distdirs.emplace_back( std::filesystem::weakly_canonical(distdir_raw)); }, "Directory to look for distfiles before fetching.") ->type_name("PATH") ->trigger_on_parse(); // run callback on all instances while parsing, // not after all parsing is done app->add_flag("--no-fetch-ssl-verify", clargs->ca_info->no_ssl_verify, "Do not perform SSL verification when fetching archives from " "remote."); app->add_option_function( "--fetch-cacert", [clargs](auto const& cacert_raw) { clargs->ca_info->ca_bundle = std::filesystem::weakly_canonical(cacert_raw); }, "CA certificate bundle to use for SSL verification when fetching " "archives from remote.") ->type_name("CA_BUNDLE"); app->add_option("--just", clargs->just_path, fmt::format("The build tool to be launched (default: {}).", kDefaultJustPath)) ->type_name("PATH"); app->add_option("--main", clargs->main, "Main repository to consider from the configuration.") ->type_name("MAIN"); app->add_option_function( "--rc", [clargs](auto const& rc_path_raw) { clargs->rc_path = std::filesystem::weakly_canonical(rc_path_raw); }, "Use just-mrrc file from custom path.") ->type_name("RCFILE"); app->add_option("--git", clargs->git_path, fmt::format("Path to the git binary. (Default: {})", kDefaultGitPath)) ->type_name("PATH"); app->add_option( "--dump-rc", clargs->dump_rc, "Dump the effective rc value.") ->type_name("PATH"); app->add_flag("--norc", clargs->norc, "Do not use any just-mrrc file."); app->add_option("-j, --jobs", clargs->jobs, "Number of jobs to run (Default: Number of cores).") ->type_name("NUM"); app->add_option_function( "-D,--defines", [clargs](auto const& d) { clargs->defines.emplace_back(d); }, "Define overlay configuration to be forwarded to the invocation of" " just, in case the subcommand supports it; otherwise ignored.") ->type_name("JSON") ->trigger_on_parse(); // run callback on all instances while parsing, // not after all parsing is done app->add_option("-r,--remote-execution-address", clargs->remote_execution_address, "Address of a remote-execution service.") ->type_name("NAME:PORT"); app->add_option( "--remote-instance-name", clargs->remote_instance_name, "Instance name of the remote-execution service (default: \"\")") ->type_name("STRING"); app->add_flag( "--compatible", clargs->compatible, "At increased computational effort, be compatible with the original " "remote build execution protocol. As the change affects identifiers, " "the flag must be used consistently for all related invocations."); app->add_option("-R,--remote-serve-address", clargs->remote_serve_address, "Address of a remote 'serve' service.") ->type_name("NAME:PORT"); app->add_flag("--fetch-absent", clargs->fetch_absent, "Do not produce absent roots. For Git repositories, try to " "fetch served commit trees from the remote execution " "endpoint before reverting to the network."); } static inline auto SetupMultiRepoLogArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-f,--log-file", [clargs](auto const& log_file_) { clargs->log_files.emplace_back(log_file_); }, "Path to local log file.") ->type_name("PATH") ->trigger_on_parse(); // run callback on all instances while parsing, // not after all parsing is done app->add_option_function>( "--log-limit", [clargs](auto const& limit) { clargs->log_limit = ToLogLevel(limit); }, fmt::format("Log limit (higher is more verbose) in interval [{},{}] " "(Default: {}).", static_cast(kFirstLogLevel), static_cast(kLastLogLevel), static_cast(kDefaultLogLevel))) ->type_name("NUM"); app->add_option_function>( "--restrict-stderr-log-limit", [clargs](auto const& limit) { clargs->restrict_stderr_log_limit = ToLogLevel(limit); }, "Restrict logging on console to the minimum of the specified " "--log-limit and this value") ->type_name("NUM"); app->add_flag("--plain-log", clargs->plain_log, "Do not use ANSI escape sequences to highlight messages."); app->add_flag( "--log-append", clargs->log_append, "Append messages to log file instead of overwriting existing."); } static inline void SetupMultiRepoSetupArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option("main-repo", clargs->sub_main, "Main repository to consider from the configuration.") ->type_name(""); app->add_flag("--all", clargs->sub_all, "Consider all repositories in the configuration."); } static inline void SetupMultiRepoFetchArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_option_function( "-o", [clargs](auto const& fetch_dir_raw) { clargs->fetch_dir = std::filesystem::weakly_canonical(fetch_dir_raw); }, "Directory to write distfiles when fetching.") ->type_name("PATH"); app->add_flag("--backup-to-remote", clargs->backup_to_remote, "Backup fetched archives to a remote CAS, if a " "remote-execution service is provided."); } static inline void SetupMultiRepoUpdateArguments( gsl::not_null const& app, gsl::not_null const& clargs) { // take all remaining args as positional app->add_option("repo", clargs->repos_to_update, "Repository to update.") ->type_name(""); } static inline void SetupMultiRepoGcArguments( gsl::not_null const& app, gsl::not_null const& clargs) { app->add_flag("--drop-only", clargs->drop_only, "Only drop old repository generations"); } static inline auto SetupMultiRepoRemoteAuthArguments( gsl::not_null const& app, gsl::not_null const& authargs) { app->add_option("--tls-ca-cert", authargs->tls_ca_cert, "Path to a TLS CA certificate that is trusted to sign the " "server certificate."); app->add_option("--tls-client-cert", authargs->tls_client_cert, "Path to the TLS client certificate."); app->add_option("--tls-client-key", authargs->tls_client_key, "Path to the TLS client key."); } #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_CLI_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/exit_codes.hpp000066400000000000000000000027401516554100600272740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_EXIT_CODES_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_EXIT_CODES_HPP #include // NOLINTNEXTLINE(performance-enum-size) enum JustMRExitCodes : std::uint16_t { kExitSuccess = 0, kExitExecError = 64, // error in execvp kExitGenericFailure = 65, // none of the known errors kExitUnknownCommand = 66, // unknown subcommand error kExitClargsError = 67, // error in parsing clargs kExitConfigError = 68, // error in parsing config kExitFetchError = 69, // error in just-mr fetch kExitUpdateError = 70, // error in just-mr update kExitSetupError = 71, // error in just-mr setup(-env) kExitBuiltinCommandFailure = 72 // error executing a built-in command }; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_EXIT_CODES_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/fetch.cpp000066400000000000000000000632571516554100600262440ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/fetch.hpp" #include #include #include #include #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/retry.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/archive_fetch_map.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/other_tools/utils/parse_archive.hpp" #include "src/other_tools/utils/parse_git_tree.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/file_locking.hpp" auto MultiRepoFetch(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoFetchArguments const& fetch_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, StorageConfig const& native_storage_config, Storage const& native_storage, std::string multi_repository_tool_name) -> int { // provide report Logger::Log(LogLevel::Info, "Performing repositories fetch"); // find fetch dir auto fetch_dir = fetch_args.fetch_dir; if (not fetch_dir) { for (auto const& d : common_args.just_mr_paths->distdirs) { if (FileSystemManager::IsDirectory(d)) { fetch_dir = std::filesystem::weakly_canonical(d); break; } } } if (not fetch_dir) { auto considered = nlohmann::json(common_args.just_mr_paths->distdirs); Logger::Log(LogLevel::Error, "No directory found to fetch to, considered {}", considered.dump()); return kExitFetchError; } auto repos = (*config)["repositories"]; if (not repos.IsNotNull()) { Logger::Log(LogLevel::Error, "Config: Mandatory key \"repositories\" missing"); return kExitFetchError; } if (not repos->IsMap()) { Logger::Log(LogLevel::Error, "Config: Value for key \"repositories\" is not a map"); return kExitFetchError; } auto fetch_repos = std::make_shared(); // repos to setup and include JustMR::Utils::DefaultReachableRepositories(repos, fetch_repos); if (not setup_args.sub_all) { auto main = common_args.main; if (not main and not fetch_repos->to_include.empty()) { main = *std::min_element(fetch_repos->to_include.begin(), fetch_repos->to_include.end()); } if (main) { JustMR::Utils::ReachableRepositories(repos, *main, fetch_repos); } std::function is_subpath = [](std::filesystem::path const& path, std::filesystem::path const& base) { return (std::filesystem::proximate(path, base) == base); }; // warn if fetch_dir is in invocation workspace if (common_args.just_mr_paths->workspace_root and is_subpath(*fetch_dir, *common_args.just_mr_paths->workspace_root)) { auto repo_desc = repos->Get(*main, Expression::none_t{}); auto repo = repo_desc->Get("repository", Expression::none_t{}); auto repo_path = repo->Get("path", Expression::none_t{}); auto repo_type = repo->Get("type", Expression::none_t{}); if (repo_path->IsString() and repo_type->IsString() and (repo_type->String() == "file")) { auto repo_path_as_path = std::filesystem::path(repo_path->String()); if (not repo_path_as_path.is_absolute()) { repo_path_as_path = std::filesystem::weakly_canonical( common_args.just_mr_paths->setup_root / repo_path_as_path); } // only warn if repo workspace differs to invocation workspace if (not is_subpath( repo_path_as_path, *common_args.just_mr_paths->workspace_root)) { Logger::Log( LogLevel::Warning, "Writing distribution files to workspace location {}, " "which is different to the workspace of the requested " "main repository {}.", nlohmann::json(fetch_dir->string()).dump(), nlohmann::json(repo_path_as_path.string()).dump()); } } } } Logger::Log(LogLevel::Info, "Fetching to {}", fetch_dir->string()); // gather all repos to be fetched std::vector archives_to_fetch{}; std::vector git_trees_to_fetch{}; archives_to_fetch.reserve( fetch_repos->to_include.size()); // pre-reserve a maximum size git_trees_to_fetch.reserve( fetch_repos->to_include.size()); // pre-reserve a maximum size for (auto const& repo_name : fetch_repos->to_include) { auto repo_desc = repos->At(repo_name); if (not repo_desc) { Logger::Log(LogLevel::Error, "Config: Missing config entry for repository {}", nlohmann::json(repo_name).dump()); return kExitFetchError; } if (not repo_desc->get()->IsMap()) { Logger::Log(LogLevel::Error, "Config: Config entry for repository {} is not a map", nlohmann::json(repo_name).dump()); return kExitFetchError; } auto repo = repo_desc->get()->At("repository"); if (repo) { auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo->get(), repos); if (not resolved_repo_desc) { Logger::Log(LogLevel::Error, "Config: Found cyclic dependency for repository {}", nlohmann::json(repo_name).dump()); return kExitFetchError; } if (not resolved_repo_desc.value()->IsMap()) { Logger::Log( LogLevel::Error, "Config: Repository {} resolves to a non-map description", nlohmann::json(repo_name).dump()); return kExitFetchError; } // get repo_type auto repo_type = (*resolved_repo_desc)->At("type"); if (not repo_type) { Logger::Log( LogLevel::Error, "Config: Mandatory key \"type\" missing for repository {}", nlohmann::json(repo_name).dump()); return kExitFetchError; } if (not repo_type->get()->IsString()) { Logger::Log(LogLevel::Error, "Config: Unsupported value {} for key \"type\" for " "repository {}", repo_type->get()->ToString(), nlohmann::json(repo_name).dump()); return kExitFetchError; } auto repo_type_str = repo_type->get()->String(); auto const checkout_type_it = kCheckoutTypeMap.find(repo_type_str); if (checkout_type_it == kCheckoutTypeMap.end()) { Logger::Log(LogLevel::Error, "Config: Unknown repository type {} for {}", nlohmann::json(repo_type_str).dump(), nlohmann::json(repo_name).dump()); return kExitFetchError; } // only do work if repo is archive or git tree type switch (checkout_type_it->second) { case CheckoutType::Archive: { auto logger = std::make_shared( [&repo_name](std::string const& msg, bool fatal) { Logger::Log( fatal ? LogLevel::Error : LogLevel::Warning, "While parsing description of repository " "{}:\n{}", nlohmann::json(repo_name).dump(), msg); }); auto archive_repo_info = ParseArchiveDescription( *resolved_repo_desc, repo_type_str, repo_name, logger); if (not archive_repo_info) { return kExitFetchError; } // only fetch if either archive is not marked absent, or if // explicitly told to fetch absent archives if (not archive_repo_info->absent or common_args.fetch_absent) { archives_to_fetch.emplace_back( archive_repo_info->archive); } } break; case CheckoutType::ForeignFile: { auto logger = std::make_shared( [&repo_name](std::string const& msg, bool fatal) { Logger::Log( fatal ? LogLevel::Error : LogLevel::Warning, "While parsing description of repository " "{}:\n{}", nlohmann::json(repo_name).dump(), msg); }); auto repo_info = ParseForeignFileDescription( *resolved_repo_desc, repo_name, logger); if (not repo_info) { return kExitFetchError; } // only fetch if either archive is not marked absent, or if // explicitly told to fetch absent archives if (not repo_info->absent or common_args.fetch_absent) { archives_to_fetch.emplace_back(repo_info->archive); } } break; case CheckoutType::GitTree: { // check "absent" pragma auto repo_desc_pragma = (*resolved_repo_desc)->At("pragma"); auto pragma_absent = (repo_desc_pragma and repo_desc_pragma->get()->IsMap()) ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); // only fetch if either archive is not marked absent, or if // explicitly told to fetch absent archives if (not pragma_absent_value or common_args.fetch_absent) { // enforce mandatory fields auto tree_info = ParseGitTree(*resolved_repo_desc, repo_name); if (not tree_info) { Logger::Log( LogLevel::Error, fmt::format("Config: {}", std::move(tree_info).error())); return kExitFetchError; } // add to list git_trees_to_fetch.emplace_back(*std::move(tree_info)); } } break; default: continue; // ignore all other repository types } } else { Logger::Log(LogLevel::Error, "Config: Missing repository description for {}", nlohmann::json(repo_name).dump()); return kExitFetchError; } } // report progress auto nr_a = archives_to_fetch.size(); auto nr_gt = git_trees_to_fetch.size(); auto str_a = fmt::format("{} {}", nr_a, nr_a == 1 ? "archive" : "archives"); auto str_gt = fmt::format("{} git {}", nr_gt, nr_gt == 1 ? "tree" : "trees"); auto fetchables = fmt::format("{}{}{}", nr_a != 0 ? str_a : std::string(), nr_a != 0 and nr_gt != 0 ? " and " : "", nr_gt != 0 ? str_gt : std::string()); if (fetchables.empty()) { Logger::Log(LogLevel::Info, "No fetch required"); } else { Logger::Log(LogLevel::Info, "Found {} to fetch", fetchables); } // setup local execution config auto local_exec_config = JustMR::Utils::CreateLocalExecutionConfig(common_args); if (not local_exec_config) { return kExitConfigError; } // pack the native local context and create api LocalContext const native_local_context{ .exec_config = &*local_exec_config, .storage_config = &native_storage_config, .storage = &native_storage}; IExecutionApi::Ptr const native_local_api = std::make_shared(&native_local_context); // pack the compatible local context, if needed std::unique_ptr compat_storage_config = nullptr; std::unique_ptr compat_storage = nullptr; std::unique_ptr compat_local_context = nullptr; std::optional compat_lock = std::nullopt; IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder::Rebuild(native_storage_config) .SetHashType(HashFunction::Type::PlainSHA256) .Build(); if (not config) { Logger::Log(LogLevel::Error, config.error()); return kExitConfigError; } compat_storage_config = std::make_unique(*std::move(config)); compat_storage = std::make_unique( Storage::Create(compat_storage_config.get())); compat_local_context = std::make_unique( LocalContext{.exec_config = &*local_exec_config, .storage_config = compat_storage_config.get(), .storage = compat_storage.get()}); // if a compatible storage is created, one must get a lock for it the // same way as done for the native one compat_lock = GarbageCollector::SharedLock(*compat_storage_config); if (not compat_lock) { Logger::Log(LogLevel::Error, "Failed to acquire compatible storage gc lock"); return kExitConfigError; } compat_local_api = std::make_shared(&*compat_local_context); } // setup the overall local api, aware of compatibility IExecutionApi::Ptr mr_local_api = std::make_shared( &native_local_context, &*native_local_api, common_args.compatible ? &*compat_local_context : nullptr, common_args.compatible ? &*compat_local_api : nullptr); // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { return kExitConfigError; } // setup the retry config auto const retry_config = CreateRetryConfig(retry_args); if (not retry_config) { return kExitConfigError; } // setup remote execution config auto const remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig( common_args.remote_execution_address, common_args.remote_serve_address, common_args.remote_instance_name ? *common_args.remote_instance_name : std::string{}); if (not remote_exec_config) { return kExitConfigError; } // create the remote api IExecutionApi::Ptr remote_api = nullptr; if (auto const address = remote_exec_config->remote_address) { auto const hash_fct = compat_local_context != nullptr ? compat_local_context->storage_config->hash_function : native_local_context.storage_config->hash_function; ExecutionConfiguration config; config.skip_cache_lookup = false; remote_api = std::make_shared(remote_exec_config->remote_instance_name, address->host, address->port, &*auth_config, &*retry_config, config, hash_fct, mr_local_api->GetTempSpace()); } bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, .retry_config = &*retry_config, .exec_config = &*remote_exec_config}; // setup the api for serving roots auto serve_config = JustMR::Utils::CreateServeConfig(common_args.remote_serve_address); if (not serve_config) { return kExitConfigError; } auto const apis = ApiBundle{.local = mr_local_api, .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr ? &*compat_local_context : &native_local_context, // defines the client's hash_function &remote_context, &apis /*unused*/); // check configuration of the serve endpoint provided if (serve) { // if we have a remote endpoint explicitly given by the user, it must // match what the serve endpoint expects if (common_args.remote_execution_address and not serve->CheckServeRemoteExecution()) { return kExitFetchError; // this check logs error on failure } // check the compatibility mode of the serve endpoint auto compatible = serve->IsCompatible(); if (not compatible) { Logger::Log(LogLevel::Warning, "Checking compatibility configuration of the provided " "serve endpoint failed. Serve endpoint ignored."); serve = std::nullopt; } if (*compatible != common_args.compatible) { Logger::Log( LogLevel::Warning, "Provided serve endpoint operates in a different compatibility " "mode than stated. Serve endpoint ignored."); serve = std::nullopt; } } // setup progress and statistics instances JustMRStatistics stats{}; JustMRProgress progress{nr_a + nr_gt}; // create async maps auto crit_git_op_ptr = std::make_shared(); auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr); auto content_cas_map = CreateContentCASMap( common_args.just_mr_paths, common_args.alternative_mirrors, common_args.ca_info, &critical_git_op_map, serve ? &*serve : nullptr, &native_storage_config, compat_storage_config != nullptr ? &*compat_storage_config : nullptr, &native_storage, compat_storage != nullptr ? &*compat_storage : nullptr, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, &progress, common_args.jobs); auto archive_fetch_map = CreateArchiveFetchMap( &content_cas_map, *fetch_dir, &native_storage, &(*apis.local), (fetch_args.backup_to_remote and has_remote_api) ? &*apis.remote : nullptr, &stats, common_args.jobs); auto import_to_git_map = CreateImportToGitMap(&critical_git_op_map, common_args.git_path->string(), *common_args.local_launcher, &native_storage_config, common_args.jobs); auto git_tree_fetch_map = CreateGitTreeFetchMap( &critical_git_op_map, &import_to_git_map, common_args.git_path->string(), *common_args.local_launcher, common_args.alternative_mirrors, serve ? &*serve : nullptr, &native_storage_config, compat_storage_config != nullptr ? &*compat_storage_config : nullptr, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, fetch_args.backup_to_remote, &progress, common_args.jobs); // set up progress observer std::atomic done{false}; std::condition_variable cv{}; auto reporter = JustMRProgressReporter::Reporter(&stats, &progress); auto observer = std::thread([reporter, &done, &cv]() { reporter(&done, &cv); }); // do the fetch bool failed_archives{false}; bool has_value_archives{false}; { TaskSystem ts{common_args.jobs}; archive_fetch_map.ConsumeAfterKeysReady( &ts, archives_to_fetch, [&has_value_archives]([[maybe_unused]] auto const& values) { has_value_archives = true; }, [&failed_archives, &multi_repository_tool_name](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While performing {} fetch:\n{}", multi_repository_tool_name, msg); failed_archives = failed_archives or fatal; }); } bool failed_git_trees{false}; bool has_value_trees{false}; { TaskSystem ts{common_args.jobs}; git_tree_fetch_map.ConsumeAfterKeysReady( &ts, git_trees_to_fetch, [&has_value_trees]([[maybe_unused]] auto const& values) { has_value_trees = true; }, [&failed_git_trees, &multi_repository_tool_name](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While performing {} fetch:\n{}", multi_repository_tool_name, msg); failed_git_trees = failed_git_trees or fatal; }); } // close progress observer done = true; cv.notify_all(); observer.join(); if (failed_archives or failed_git_trees) { return kExitFetchError; } if (not has_value_archives or not has_value_trees) { DetectAndReportPending( "fetch archives", archive_fetch_map, kArchiveContentPrinter); DetectAndReportPending( "fetch trees", git_tree_fetch_map, kGitTreeInfoPrinter); return kExitFetchError; } // report success Logger::Log(LogLevel::Info, "Fetch completed"); return kExitSuccess; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/fetch.hpp000066400000000000000000000034121516554100600262340ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_FETCH_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_FETCH_HPP #include #include #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Fetching of distfiles for a multi-repository build. [[nodiscard]] auto MultiRepoFetch(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoFetchArguments const& fetch_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, StorageConfig const& storage_config, Storage const& storage, std::string multi_repository_tool_name) -> int; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_FETCH_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/launch.cpp000066400000000000000000000425231516554100600264160ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/launch.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include // for errno #include #include // for strerror() #include #include #include #include #include #include #include #include #include #include #include #include "fmt/chrono.h" #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/repository_garbage_collector.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/setup.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/file_locking.hpp" #include "src/utils/cpp/path.hpp" auto CallJust(std::optional const& config_file, InvocationLogArguments const& invocation_log, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoLogArguments const& log_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, ForwardOnlyArguments const& launch_fwd, StorageConfig const& storage_config, Storage const& storage, bool forward_build_root, std::string const& multi_repo_tool_name) -> int { // check if subcmd_name can be taken from additional args auto additional_args_offset = 0U; auto subcommand = just_cmd_args.subcmd_name; if (not subcommand and not just_cmd_args.additional_just_args.empty()) { subcommand = just_cmd_args.additional_just_args[0]; additional_args_offset++; } bool use_config{false}; bool use_build_root{false}; bool use_launcher{false}; bool supports_defines{false}; bool supports_remote{false}; bool supports_remote_properties{false}; bool supports_serve{false}; bool supports_dispatch{false}; bool does_build{false}; std::optional> mr_config_pair{ std::nullopt}; // If gc locks are needed, ensure to keep them alive also for the exec call std::optional lock{}; std::optional repo_lock{}; if (subcommand and kKnownJustSubcommands.contains(*subcommand)) { auto const& flags = kKnownJustSubcommands.at(*subcommand); // Read the config file if needed if (flags.config) { repo_lock = RepositoryGarbageCollector::SharedLock(storage_config); if (not repo_lock) { return kExitGenericFailure; } lock = GarbageCollector::SharedLock(storage_config); if (not lock) { return kExitGenericFailure; } auto config = JustMR::Utils::ReadConfiguration( config_file, common_args.absent_repository_file); use_config = true; mr_config_pair = MultiRepoSetup(config, common_args, setup_args, just_cmd_args, auth_args, retry_args, storage_config, storage, /*interactive=*/false, multi_repo_tool_name); if (not mr_config_pair) { Logger::Log(LogLevel::Error, "Failed to setup config for calling \"{} {}\"", common_args.just_path ? common_args.just_path->string() : kDefaultJustPath, *subcommand); return kExitSetupError; } } use_build_root = flags.build_root; use_launcher = flags.launch; supports_defines = flags.defines; supports_remote = flags.remote; supports_remote_properties = flags.remote_props; supports_serve = flags.serve; supports_dispatch = flags.dispatch; does_build = flags.does_build; } // build just command std::vector cmd = {common_args.just_path->string()}; if (subcommand) { cmd.emplace_back(*subcommand); } if (use_config) { cmd.emplace_back("-C"); cmd.emplace_back(mr_config_pair->first.string()); } if (use_build_root and forward_build_root) { cmd.emplace_back("--local-build-root"); cmd.emplace_back(*common_args.just_mr_paths->root); } if (use_launcher and common_args.local_launcher and (common_args.local_launcher != kDefaultLauncher)) { cmd.emplace_back("--local-launcher"); cmd.emplace_back(nlohmann::json(*common_args.local_launcher).dump()); } // forward logging arguments if (not log_args.log_files.empty()) { cmd.emplace_back("--log-append"); for (auto const& log_file : log_args.log_files) { cmd.emplace_back("-f"); cmd.emplace_back(log_file.string()); } } if (log_args.log_limit and *log_args.log_limit != kDefaultLogLevel) { cmd.emplace_back("--log-limit"); cmd.emplace_back( std::to_string(static_cast>( *log_args.log_limit))); } if (log_args.restrict_stderr_log_limit) { cmd.emplace_back("--restrict-stderr-log-limit"); cmd.emplace_back( std::to_string(static_cast>( *log_args.restrict_stderr_log_limit))); } if (log_args.plain_log) { cmd.emplace_back("--plain-log"); } if (supports_defines) { if (just_cmd_args.config) { cmd.emplace_back("-c"); cmd.emplace_back(just_cmd_args.config->string()); } auto overlay_config = Configuration(); for (auto const& s : common_args.defines) { try { auto map = Expression::FromJson(nlohmann::json::parse(s)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "Defines entry {} does not contain a map.", nlohmann::json(s).dump()); std::exit(kExitClargsError); } overlay_config = overlay_config.Update(map); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing defines entry {} failed with error:\n{}", nlohmann::json(s).dump(), e.what()); std::exit(kExitClargsError); } } if (not overlay_config.Expr()->Map().empty()) { cmd.emplace_back("-D"); cmd.emplace_back(overlay_config.ToString()); } } // forward remote execution and mutual TLS arguments if (supports_remote) { if (common_args.compatible) { cmd.emplace_back("--compatible"); } if (common_args.remote_execution_address) { cmd.emplace_back("-r"); cmd.emplace_back(*common_args.remote_execution_address); if (common_args.remote_instance_name) { cmd.emplace_back("--remote-instance-name"); cmd.emplace_back(*common_args.remote_instance_name); } } if (auth_args.tls_ca_cert) { cmd.emplace_back("--tls-ca-cert"); cmd.emplace_back(auth_args.tls_ca_cert->string()); } if (auth_args.tls_client_cert) { cmd.emplace_back("--tls-client-cert"); cmd.emplace_back(auth_args.tls_client_cert->string()); } if (auth_args.tls_client_key) { cmd.emplace_back("--tls-client-key"); cmd.emplace_back(auth_args.tls_client_key->string()); } if (retry_args.max_attempts) { cmd.emplace_back("--max-attempts"); cmd.emplace_back(std::to_string(*retry_args.max_attempts)); } if (retry_args.initial_backoff_seconds) { cmd.emplace_back("--initial-backoff-seconds"); cmd.emplace_back( std::to_string(*retry_args.initial_backoff_seconds)); } if (retry_args.max_backoff_seconds) { cmd.emplace_back("--max-backoff-seconds"); cmd.emplace_back(std::to_string(*retry_args.max_backoff_seconds)); } } if (supports_dispatch and just_cmd_args.endpoint_configuration) { cmd.emplace_back("--endpoint-configuration"); cmd.emplace_back(*just_cmd_args.endpoint_configuration); } if (supports_serve and common_args.remote_serve_address) { cmd.emplace_back("-R"); cmd.emplace_back(*common_args.remote_serve_address); } // forward-only arguments, still to come before the just-arguments if (supports_remote_properties) { for (auto const& prop : launch_fwd.remote_execution_properties) { cmd.emplace_back("--remote-execution-property"); cmd.emplace_back(prop); } } // add args read from just-mrrc if (subcommand and just_cmd_args.just_args.contains(*subcommand)) { for (auto const& subcmd_arg : just_cmd_args.just_args.at(*subcommand)) { cmd.emplace_back(subcmd_arg); } } std::optional log_dir = std::nullopt; auto invocation_time = std::time(nullptr); // Check invocation logging if (invocation_log.directory) { auto dir = *invocation_log.directory; if (invocation_log.project_id) { if (not IsValidFileName(*invocation_log.project_id)) { Logger::Log(LogLevel::Error, "Invalid file name for poject id: {}", nlohmann::json(*invocation_log.project_id).dump()); std::exit(kExitClargsError); } dir = dir / *invocation_log.project_id; } else { dir = dir / "unknown"; } std::string uuid = CreateUUID(); auto invocation_id = fmt::format( "{:%Y-%m-%d-%H:%M:%S}-{}", fmt::gmtime(invocation_time), uuid); dir = dir / invocation_id; if (FileSystemManager::CreateDirectoryExclusive(dir)) { if (invocation_log.invocation_msg) { Logger::Log(LogLevel::Info, "{}{}", *invocation_log.invocation_msg, invocation_id); } Logger::Log( LogLevel::Info, "Invocation logged at {}", dir.string()); log_dir = dir; } else { Logger::Log(LogLevel::Warning, "Failed to create directory {} for invocation logging", nlohmann::json(dir.string()).dump()); } } // invocation-specific if (log_dir and supports_remote_properties) { if (invocation_log.graph_file) { if (not IsValidFileName(*invocation_log.graph_file)) { Logger::Log(LogLevel::Error, "Invalid file name for option --dump-graph: {}", nlohmann::json(*invocation_log.graph_file).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-graph"); cmd.emplace_back(*log_dir / *invocation_log.graph_file); } if (invocation_log.graph_file_plain) { if (not IsValidFileName(*invocation_log.graph_file_plain)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-plain-graph: {}", nlohmann::json(*invocation_log.graph_file_plain).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-plain-graph"); cmd.emplace_back(*log_dir / *invocation_log.graph_file_plain); } if (invocation_log.dump_artifacts_to_build) { if (not IsValidFileName(*invocation_log.dump_artifacts_to_build)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-artifacts_to_build: " "{}", nlohmann::json(*invocation_log.dump_artifacts_to_build) .dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-artifacts-to-build"); cmd.emplace_back(*log_dir / *invocation_log.dump_artifacts_to_build); } if (does_build and invocation_log.dump_artifacts) { if (not IsValidFileName(*invocation_log.dump_artifacts)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-artifacts: {}", nlohmann::json(*invocation_log.dump_artifacts).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-artifacts"); cmd.emplace_back(*log_dir / *invocation_log.dump_artifacts); } if (invocation_log.profile) { if (not IsValidFileName(*invocation_log.profile)) { Logger::Log(LogLevel::Error, "Invalid file name for option --profile: {}", nlohmann::json(*invocation_log.profile).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--profile"); cmd.emplace_back(*log_dir / *invocation_log.profile); } } // add (remaining) args given by user as clargs for (auto it = just_cmd_args.additional_just_args.begin() + additional_args_offset; it != just_cmd_args.additional_just_args.end(); ++it) { cmd.emplace_back(*it); } // Write invocation metadata, if requested if (log_dir and invocation_log.metadata) { if (not IsValidFileName(*invocation_log.metadata)) { Logger::Log(LogLevel::Error, "Invlaid file name for metadata file: {}", nlohmann::json(*invocation_log.metadata).dump()); std::exit(kExitClargsError); } auto meta = nlohmann::json::object(); meta["time"] = invocation_time; if (mr_config_pair) { meta["configuration"] = mr_config_pair->second; } meta["cmdline"] = cmd; if (not invocation_log.context_vars.empty()) { auto context = nlohmann::json::object(); for (auto const& env_var : invocation_log.context_vars) { auto* env_value = std::getenv(env_var.c_str()); context[env_var] = env_value != nullptr ? env_value : nlohmann::json{}; } meta["context"] = context; } // "configuration" -- the blob-identifier of the multi-repo // configuration auto file_name = *log_dir / *invocation_log.metadata; if (not FileSystemManager::WriteFile(meta.dump(2), file_name)) { Logger::Log(LogLevel::Warning, "Failed to write metadata file {}.", nlohmann::json(file_name).dump()); } } Logger::Log( LogLevel::Info, "Setup finished, exec {}", nlohmann::json(cmd).dump()); // create argv std::vector argv{}; std::transform(std::begin(cmd), std::end(cmd), std::back_inserter(argv), [](auto& str) { return str.data(); }); argv.push_back(nullptr); // run execvp; will only return if failure [[maybe_unused]] auto res = execvp(argv[0], static_cast(argv.data())); // execvp returns only if command errored out Logger::Log(LogLevel::Error, "execvp failed with:\n{}", strerror(errno)); return kExitExecError; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/launch.hpp000066400000000000000000000032661516554100600264240ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_LAUNCH_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_LAUNCH_HPP #include #include #include #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Runs execvp for configured command. Only returns if execvp fails. [[nodiscard]] auto CallJust( std::optional const& config_file, InvocationLogArguments const& invocation_log, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoLogArguments const& log_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, ForwardOnlyArguments const& launch_fwd, StorageConfig const& storage_config, Storage const& storage, bool forward_build_root, std::string const& multi_repo_tool_name) -> int; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_LAUNCH_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/main.cpp000066400000000000000000000451051516554100600260670ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include #include #include #include "CLI/CLI.hpp" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" #include "src/buildtool/logging/log_sink_file.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/version.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/repository_garbage_collector.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/fetch.hpp" #include "src/other_tools/just_mr/launch.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/rc.hpp" #include "src/other_tools/just_mr/setup.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" #include "src/other_tools/just_mr/update.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/expected.hpp" namespace { /// \brief Setup arguments for just-mr itself, common to all subcommands. void SetupCommonCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupMultiRepoCommonArguments(app, &clargs->common); SetupMultiRepoLogArguments(app, &clargs->log); SetupMultiRepoRemoteAuthArguments(app, &clargs->auth); SetupRetryArguments(app, &clargs->retry); } /// \brief Setup arguments for subcommand "just-mr fetch". void SetupFetchCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupMultiRepoSetupArguments(app, &clargs->setup); SetupMultiRepoFetchArguments(app, &clargs->fetch); } /// \brief Setup arguments for subcommand "just-mr update". void SetupUpdateCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupMultiRepoUpdateArguments(app, &clargs->update); } /// \brief Setup arguments for subcommand "just-mr gc-repo". void SetupUpdateGcArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupMultiRepoGcArguments(app, &clargs->gc); } /// \brief Setup arguments for subcommand "just-mr setup" and /// "just-mr setup-env". void SetupSetupCommandArguments( gsl::not_null const& app, gsl::not_null const& clargs) { SetupMultiRepoSetupArguments(app, &clargs->setup); } [[nodiscard]] auto ParseCommandLineArguments(int argc, char const* const* argv) -> CommandLineArguments { CLI::App app( "just-mr, a multi-repository configuration tool and launcher for the " "build tool"); app.option_defaults()->take_last(); auto* cmd_mrversion = app.add_subcommand( "mrversion", "Print version information in JSON format of this tool."); auto* cmd_setup = app.add_subcommand( "setup", "Setup and generate configuration for the build tool"); auto* cmd_setup_env = app.add_subcommand( "setup-env", "Setup without workspace root for the main repository."); auto* cmd_fetch = app.add_subcommand("fetch", "Fetch and store distribution files."); auto* cmd_update = app.add_subcommand( "update", "Advance Git commit IDs and print updated just-mr configuration."); auto* cmd_do = app.add_subcommand( "do", "Canonical way of specifying subcommands to be launched."); auto* cmd_gc_repo = app.add_subcommand( "gc-repo", "Perform garbage collection on the repository roots."); cmd_do->set_help_flag(); // disable help flag // define just subcommands std::vector cmd_just_subcmds{}; cmd_just_subcmds.reserve(kKnownJustSubcommands.size()); for (auto const& known_subcmd : kKnownJustSubcommands) { auto* subcmd = app.add_subcommand(known_subcmd.first, "Run setup and launch the \"" + known_subcmd.first + "\" subcommand."); subcmd->set_help_flag(); // disable help flag cmd_just_subcmds.emplace_back(subcmd); } app.require_subcommand(1); CommandLineArguments clargs; // first, set the common arguments for just-mr itself SetupCommonCommandArguments(&app, &clargs); // then, set the arguments for each subcommand SetupSetupCommandArguments(cmd_setup, &clargs); SetupSetupCommandArguments(cmd_setup_env, &clargs); SetupFetchCommandArguments(cmd_fetch, &clargs); SetupUpdateCommandArguments(cmd_update, &clargs); SetupUpdateGcArguments(cmd_gc_repo, &clargs); // for 'just' calls, allow extra arguments cmd_do->allow_extras(); for (auto const& sub_cmd : cmd_just_subcmds) { sub_cmd->allow_extras(); } try { app.parse(argc, argv); } catch (CLI::Error& e) { // CLI11 throws for things like --help calls for them to be handled // separately by parse callers. In this case it nevertheless sets the // error code to 0 (success). auto const err = app.exit(e); std::exit(err == 0 ? kExitSuccess : kExitClargsError); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Command line parse error: {}", ex.what()); std::exit(kExitClargsError); } if (*cmd_mrversion) { clargs.cmd = SubCommand::kMRVersion; } else if (*cmd_setup) { clargs.cmd = SubCommand::kSetup; } else if (*cmd_setup_env) { clargs.cmd = SubCommand::kSetupEnv; } else if (*cmd_fetch) { clargs.cmd = SubCommand::kFetch; } else if (*cmd_update) { clargs.cmd = SubCommand::kUpdate; } else if (*cmd_gc_repo) { clargs.cmd = SubCommand::kGcRepo; } else if (*cmd_do) { clargs.cmd = SubCommand::kJustDo; // get remaining args clargs.just_cmd.additional_just_args = cmd_do->remaining(); } else { for (auto const& sub_cmd : cmd_just_subcmds) { if (*sub_cmd) { clargs.cmd = SubCommand::kJustSubCmd; clargs.just_cmd.subcmd_name = sub_cmd->get_name(); // get name of subcommand // get remaining args clargs.just_cmd.additional_just_args = sub_cmd->remaining(); break; // no need to go further } } } return clargs; } void SetupDefaultLogging() { LogConfig::SetLogLimit(kDefaultLogLevel); LogConfig::SetSinks({LogSinkCmdLine::CreateFactory()}); } void SetupLogging(MultiRepoLogArguments const& clargs) { if (clargs.log_limit) { LogConfig::SetLogLimit(*clargs.log_limit); } else { LogConfig::SetLogLimit(kDefaultLogLevel); } LogConfig::SetSinks({LogSinkCmdLine::CreateFactory( not clargs.plain_log, clargs.restrict_stderr_log_limit)}); for (auto const& log_file : clargs.log_files) { LogConfig::AddSink(LogSinkFile::CreateFactory( log_file, clargs.log_append ? LogSinkFile::Mode::Append : LogSinkFile::Mode::Overwrite)); } } [[nodiscard]] auto CreateStorageConfig(MultiRepoCommonArguments const& args, HashFunction::Type hash_type) noexcept -> std::optional { StorageConfig::Builder builder; if (args.just_mr_paths->root.has_value()) { builder.SetBuildRoot(*args.just_mr_paths->root); } builder.SetHashType(hash_type); // As just-mr does not require the TargetCache, we do not need to set any of // the remote execution fields for the backend description. auto config = builder.Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } } // namespace auto main(int argc, char* argv[]) -> int { SetupDefaultLogging(); std::string my_name{}; if (argc > 0) { try { // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) my_name = std::filesystem::path(argv[0]).filename().string(); } catch (...) { // ignore, as my_name is only used for error messages my_name.clear(); } } try { // get the user-defined arguments auto arguments = ParseCommandLineArguments(argc, argv); if (arguments.cmd == SubCommand::kMRVersion) { std::cout << version() << std::endl; return kExitSuccess; } SetupLogging(arguments.log); // Parse rc file, if given, and returns any configuration file found in // known locations, with the setup root updated accordingly auto config_file = ReadJustMRRC(&arguments); // As the rc file can contain logging parameters, reset the logging // configuration SetupLogging(arguments.log); // An explicitly given configuration file path wins. In this case, the // default setup root must be used if (arguments.common.repository_config) { config_file = arguments.common.repository_config; arguments.common.just_mr_paths->setup_root = kDefaultSetupRoot; } // if optional args were not read from just-mrrc or given by user, use // the defaults if (not arguments.common.just_path) { arguments.common.just_path = kDefaultJustPath; } if (not arguments.common.git_path) { arguments.common.git_path = kDefaultGitPath; } bool forward_build_root = true; if (not arguments.common.just_mr_paths->root) { forward_build_root = false; arguments.common.just_mr_paths->root = std::filesystem::weakly_canonical(kDefaultBuildRoot); } if (not arguments.common.checkout_locations_file and FileSystemManager::IsFile(std::filesystem::weakly_canonical( kDefaultCheckoutLocationsFile))) { arguments.common.checkout_locations_file = std::filesystem::weakly_canonical( kDefaultCheckoutLocationsFile); } if (arguments.common.just_mr_paths->distdirs.empty()) { arguments.common.just_mr_paths->distdirs.emplace_back( kDefaultDistdirs); } // read checkout locations and alternative mirrors if (arguments.common.checkout_locations_file) { try { std::ifstream ifs(*arguments.common.checkout_locations_file); auto checkout_locations_json = nlohmann::json::parse(ifs); arguments.common.just_mr_paths->git_checkout_locations = checkout_locations_json .value("checkouts", nlohmann::json::object()) .value("git", nlohmann::json::object()); arguments.common.alternative_mirrors->local_mirrors = checkout_locations_json.value("local mirrors", nlohmann::json::object()); arguments.common.alternative_mirrors->preferred_hostnames = checkout_locations_json.value("preferred hostnames", nlohmann::json::array()); arguments.common.alternative_mirrors->extra_inherit_env = checkout_locations_json.value("extra inherit env", nlohmann::json::array()); } catch (std::exception const& e) { Logger::Log( LogLevel::Error, "Parsing checkout locations file {} failed with error:\n{}", arguments.common.checkout_locations_file->string(), e.what()); std::exit(kExitConfigError); } } // append explicitly-given distdirs arguments.common.just_mr_paths->distdirs.insert( arguments.common.just_mr_paths->distdirs.end(), arguments.common.explicit_distdirs.begin(), arguments.common.explicit_distdirs.end()); // Setup LocalStorageConfig to store the local_build_root properly // and make the cas and git cache roots available. A native storage is // always instantiated, while a compatible one only if needed. auto const native_storage_config = CreateStorageConfig(arguments.common, HashFunction::Type::GitSHA1); if (not native_storage_config) { Logger::Log(LogLevel::Error, "Failed to configure local build root."); return kExitGenericFailure; } if (arguments.cmd == SubCommand::kGcRepo) { return RepositoryGarbageCollector::TriggerGarbageCollection( *native_storage_config, arguments.gc.drop_only) ? kExitSuccess : kExitBuiltinCommandFailure; } auto const native_storage = Storage::Create(&*native_storage_config); // check for conflicts in main repo name if ((not arguments.setup.sub_all) and arguments.common.main and arguments.setup.sub_main and (arguments.common.main != arguments.setup.sub_main)) { Logger::Log(LogLevel::Warning, "Conflicting options for main repository, selecting {}", *arguments.setup.sub_main); } if (arguments.setup.sub_main) { arguments.common.main = arguments.setup.sub_main; } // check for errors in setting up local launcher arg if (not arguments.common.local_launcher) { Logger::Log(LogLevel::Error, "Failed to configure local execution."); return kExitGenericFailure; } /** * The current implementation of libgit2 uses pthread_key_t incorrectly * on POSIX systems to handle thread-specific data, which requires us to * explicitly make sure the main thread is the first one to call * git_libgit2_init. Future versions of libgit2 will hopefully fix this. */ GitContext::Create(); // Run subcommands known to just and `do` if (arguments.cmd == SubCommand::kJustDo or arguments.cmd == SubCommand::kJustSubCmd) { return CallJust(config_file, arguments.invocation_log, arguments.common, arguments.setup, arguments.just_cmd, arguments.log, arguments.auth, arguments.retry, arguments.launch_fwd, *native_storage_config, native_storage, forward_build_root, my_name); } auto repo_lock = RepositoryGarbageCollector::SharedLock(*native_storage_config); if (not repo_lock) { return kExitGenericFailure; } auto lock = GarbageCollector::SharedLock(*native_storage_config); if (not lock) { return kExitGenericFailure; } // The remaining options all need the config file auto config = JustMR::Utils::ReadConfiguration( config_file, arguments.common.absent_repository_file); // Run subcommand `setup` or `setup-env` if (arguments.cmd == SubCommand::kSetup or arguments.cmd == SubCommand::kSetupEnv) { auto mr_config_path = MultiRepoSetup( config, arguments.common, arguments.setup, arguments.just_cmd, arguments.auth, arguments.retry, *native_storage_config, native_storage, /*interactive=*/(arguments.cmd == SubCommand::kSetupEnv), my_name); // dump resulting config to stdout if (not mr_config_path) { return kExitSetupError; } // report success Logger::Log(LogLevel::Info, "Setup completed"); // print config file to stdout std::cout << mr_config_path->first.string() << std::endl; return kExitSuccess; } // Run subcommand `update` if (arguments.cmd == SubCommand::kUpdate) { return MultiRepoUpdate(config, arguments.common, arguments.update, *native_storage_config, my_name); } // Run subcommand `fetch` if (arguments.cmd == SubCommand::kFetch) { return MultiRepoFetch(config, arguments.common, arguments.setup, arguments.fetch, arguments.auth, arguments.retry, *native_storage_config, native_storage, my_name); } // Unknown subcommand should fail Logger::Log(LogLevel::Error, "Unknown subcommand provided."); return kExitUnknownCommand; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Caught exception with message: {}", ex.what()); } return kExitGenericFailure; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/mirrors.hpp000066400000000000000000000161361516554100600266470ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_MIRRORS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_MIRRORS_HPP #include #include #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/other_tools/utils/curl_url_handle.hpp" struct Mirrors { nlohmann::json local_mirrors; // maps URLs to list of local mirrors nlohmann::json preferred_hostnames; // list of mirror hostnames nlohmann::json extra_inherit_env; }; using MirrorsPtr = std::shared_ptr; namespace MirrorsUtils { /// \brief Get the list of local mirrors for given primary URL. [[nodiscard]] static inline auto GetLocalMirrors( MirrorsPtr const& additional_mirrors, std::string const& repo_url) noexcept -> std::vector { try { if (additional_mirrors->local_mirrors.contains(repo_url)) { auto const& arr = additional_mirrors->local_mirrors[repo_url]; if (arr.is_array()) { std::vector res{}; res.reserve(arr.size()); for (auto const& [_, val] : arr.items()) { if (val.is_string()) { res.emplace_back(val.get()); } else { Logger::Log(LogLevel::Warning, "Retrieving additional mirrors: found " "non-string list entry {}", val.dump()); return {}; } } return res; } Logger::Log( LogLevel::Warning, "Retrieving additional mirrors: found non-list value {}", arr.dump()); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Retrieving additional mirrors failed with:\n{}", ex.what()); } return {}; } /// \brief Get the list of preferred hostnames for non-local fetches. [[nodiscard]] static inline auto GetPreferredHostnames( MirrorsPtr const& additional_mirrors) noexcept -> std::vector { try { std::vector res{}; res.reserve(additional_mirrors->preferred_hostnames.size()); for (auto const& [_, val] : additional_mirrors->preferred_hostnames.items()) { if (val.is_string()) { res.emplace_back(val.get()); } else { Logger::Log(LogLevel::Warning, "Retrieving preferred mirrors: found non-string " "list entry {}", val.dump()); return {}; } } return res; } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Retrieving preferred mirrors failed with:\n{}", ex.what()); return {}; }; } /// \brief Get the list of extra variables to inherit. [[nodiscard]] static inline auto GetInheritEnv( MirrorsPtr const& additional_mirrors, std::vector const& base) noexcept -> std::vector { try { std::vector res(base); res.reserve(additional_mirrors->extra_inherit_env.size() + base.size()); for (auto const& [_, val] : additional_mirrors->extra_inherit_env.items()) { if (val.is_string()) { res.emplace_back(val.get()); } else { Logger::Log( LogLevel::Warning, "Retrieving extra variables to inherit: found non-string " "list entry {}", val.dump()); } } return res; } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, "Retrieving extra environment variables to inherit failed " "with:\n{}", ex.what()); return {}; }; } /// \brief Sort mirrors by the order of given hostnames. [[nodiscard]] static inline auto SortByHostname( std::vector const& mirrors, std::vector const& hostnames) -> std::vector { using map_t = std::unordered_map>; map_t mirrors_by_hostname{}; mirrors_by_hostname.reserve(hostnames.size() + 1); // initialize map with known hostnames std::transform( hostnames.begin(), hostnames.end(), std::inserter(mirrors_by_hostname, mirrors_by_hostname.end()), [](auto const& hostname) -> typename map_t::value_type { return {hostname, {/*empty*/}}; }); // fill mirrors list per hostname for (auto const& mirror : mirrors) { auto hostname = CurlURLHandle::GetHostname(mirror).value_or(std::string{}); auto it = mirrors_by_hostname.find(hostname); if (it != mirrors_by_hostname.end()) { it->second.emplace_back(mirror); } else { // add missing or unknown hostnames to fallback list with key "" mirrors_by_hostname[""].emplace_back(mirror); } } std::vector ordered{}; ordered.reserve(mirrors.size()); // first, add mirrors in the order defined by hostnames for (auto const& hostname : hostnames) { auto it = mirrors_by_hostname.find(hostname); if (it != mirrors_by_hostname.end()) { auto& list = it->second; ordered.insert(ordered.end(), std::make_move_iterator(list.begin()), std::make_move_iterator(list.end())); // clear for safe revisit in case hostnames contains any duplicates list.clear(); } } // second, append remaining mirrors from fallback list with key "" auto it = mirrors_by_hostname.find(""); if (it != mirrors_by_hostname.end()) { auto& list = it->second; ordered.insert(ordered.end(), std::make_move_iterator(list.begin()), std::make_move_iterator(list.end())); } return ordered; } } // namespace MirrorsUtils #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_MIRRORS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/000077500000000000000000000000001516554100600303675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/TARGETS000066400000000000000000000020731516554100600314250ustar00rootroot00000000000000{ "statistics": { "type": ["@", "rules", "CC", "library"] , "name": ["statistics"] , "hdrs": ["statistics.hpp"] , "stage": ["src", "other_tools", "just_mr", "progress_reporting"] } , "progress": { "type": ["@", "rules", "CC", "library"] , "name": ["progress"] , "hdrs": ["progress.hpp"] , "stage": ["src", "other_tools", "just_mr", "progress_reporting"] , "deps": [["src/buildtool/progress_reporting", "task_tracker"]] } , "progress_reporter": { "type": ["@", "rules", "CC", "library"] , "name": ["progress_reporter"] , "hdrs": ["progress_reporter.hpp"] , "srcs": ["progress_reporter.cpp"] , "stage": ["src", "other_tools", "just_mr", "progress_reporting"] , "deps": [ "progress" , "statistics" , ["@", "gsl", "", "gsl"] , ["src/buildtool/progress_reporting", "base_progress_reporter"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "task_tracker"] ] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/progress.hpp000066400000000000000000000024451516554100600327510ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_HPP #include #include "src/buildtool/progress_reporting/task_tracker.hpp" class JustMRProgress final { public: explicit JustMRProgress(std::size_t total) noexcept : total_{total} {}; [[nodiscard]] auto TaskTracker() noexcept -> TaskTracker& { return task_tracker_; } [[nodiscard]] auto GetTotal() const noexcept -> std::size_t { return total_; } private: ::TaskTracker task_tracker_{}; std::size_t total_ = 0; }; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/progress_reporter.cpp000066400000000000000000000046511516554100600346670ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" auto JustMRProgressReporter::Reporter( gsl::not_null const& stats, gsl::not_null const& progress) noexcept -> progress_reporter_t { return BaseProgressReporter::Reporter([stats, progress]() { auto const total = progress->GetTotal(); auto const local = stats->LocalPathsCounter(); auto const cached = stats->CacheHitsCounter(); auto const computed = stats->ComputedCounter(); auto const run = stats->ExecutedCounter(); auto const active = progress->TaskTracker().Active(); auto const sample = progress->TaskTracker().Sample(); auto msg = fmt::format("{} computed, {} local, {} cached, {} done", computed, local, cached, run); if ((active > 0) and not sample.empty()) { msg = fmt::format("{}; {} fetches ({}{})", msg, active, nlohmann::json(sample).dump(), active > 1 ? ", ..." : ""); } constexpr int kOneHundred{100}; int progress = kOneHundred; // default if no work has to be done if (auto const noops = cached + local + computed; noops < total) { progress = static_cast(run * kOneHundred / (total - noops)); } Logger::Log(LogLevel::Progress, "[{:3}%] {}", progress, msg); }); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/progress_reporter.hpp000066400000000000000000000024751516554100600346760ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP #include "gsl/gsl" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" class JustMRProgressReporter final { public: [[nodiscard]] static auto Reporter( gsl::not_null const& stats, gsl::not_null const& progress) noexcept -> progress_reporter_t; }; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_REPORTER_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/progress_reporting/statistics.hpp000066400000000000000000000036401516554100600332750ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_STATISTICS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_STATISTICS_HPP #include #include class JustMRStatistics final { public: void IncrementLocalPathsCounter() noexcept { ++num_local_paths_; } void IncrementCacheHitsCounter() noexcept { ++num_cache_hits_; } void IncrementComputedCounter() noexcept { ++num_computed_; } void IncrementExecutedCounter() noexcept { ++num_executed_; } [[nodiscard]] auto LocalPathsCounter() const noexcept -> size_t { return num_local_paths_; } [[nodiscard]] auto CacheHitsCounter() const noexcept -> size_t { return num_cache_hits_; } [[nodiscard]] auto ComputedCounter() const noexcept -> size_t { return num_computed_; } [[nodiscard]] auto ExecutedCounter() const noexcept -> size_t { return num_executed_; } private: std::atomic num_local_paths_ = 0; // roots that are real paths std::atomic num_cache_hits_ = 0; // no-ops std::atomic num_computed_ = 0; // no work to be done, as it is computed std::atomic num_executed_ = 0; // actual work done }; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_STATISTICS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/rc.cpp000066400000000000000000000714741516554100600255570ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/rc.hpp" #include #include #include #include #include #include #include #include #include // std::move #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/location.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/rc_merge.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/expected.hpp" namespace { /// \brief Overlay for ReadLocationObject accepting an ExpressionPtr and that /// can std::exit [[nodiscard]] auto ReadLocation( ExpressionPtr const& location, std::optional const& ws_root) -> std::optional> { if (location.IsNotNull()) { auto res = ReadLocationObject(location->ToJson(), ws_root); if (not res) { Logger::Log(LogLevel::Error, res.error()); std::exit(kExitConfigError); } return *res; } return std::nullopt; } /// \brief Overlay of ReadLocationObject that can std::exit [[nodiscard]] auto ReadLocation( nlohmann::json const& location, std::optional const& ws_root) -> std::optional> { auto res = ReadLocationObject(location, ws_root); if (not res) { Logger::Log(LogLevel::Error, res.error()); std::exit(kExitConfigError); } return *res; } [[nodiscard]] auto ReadOptionalLocationList( ExpressionPtr const& location_list, std::optional const& ws_root, std::string const& argument_name) -> std::optional { if (location_list->IsNone()) { return std::nullopt; } if (not location_list->IsList()) { Logger::Log(LogLevel::Error, "Argument {} has to be a list, but found {}", argument_name, location_list->ToString()); std::exit(kExitConfigError); } for (auto const& location : location_list->List()) { auto p = ReadLocation(location, ws_root); if (p and FileSystemManager::IsFile(p->first)) { return p->first; } } return std::nullopt; } [[nodiscard]] auto ObtainRCConfig( gsl::not_null const& clargs) -> Configuration { Configuration rc_config{}; auto rc_path = clargs->common.rc_path; // set default if rcpath not given if (not clargs->common.norc) { if (not rc_path) { rc_path = std::filesystem::weakly_canonical(kDefaultRCPath); } else { if (not FileSystemManager::IsFile(*rc_path)) { Logger::Log(LogLevel::Error, "Cannot read RC file {}.", rc_path->string()); std::exit(kExitConfigError); } } if (FileSystemManager::IsFile(*rc_path)) { // json::parse may throw try { std::ifstream fs(*rc_path); auto map = Expression::FromJson(nlohmann::json::parse(fs)); if (not map->IsMap()) { Logger::Log( LogLevel::Error, "In RC file {}: expected an object but found:\n{}", rc_path->string(), map->ToString()); std::exit(kExitConfigError); } rc_config = Configuration{map}; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing RC file {} as JSON failed with error:\n{}", rc_path->string(), e.what()); std::exit(kExitConfigError); } } } return rc_config; } } // namespace /// \brief Read just-mrrc file and set up various configs. Return the path to /// the repository config file, if any is provided. [[nodiscard]] auto ReadJustMRRC( gsl::not_null const& clargs) -> std::optional { Configuration rc_config = ObtainRCConfig(clargs); // Merge in the rc-files to overlay auto extra_rc_files = rc_config["rc files"]; if (not extra_rc_files->IsNone()) { if (not extra_rc_files->IsList()) { Logger::Log( LogLevel::Error, "'rc files' has to be a list of location objects, but found {}", extra_rc_files->ToString()); std::exit(kExitConfigError); } for (auto const& entry : extra_rc_files->List()) { auto extra_rc_location = ReadLocation( entry, clargs->common.just_mr_paths->workspace_root); if (extra_rc_location) { auto const& extra_rc_path = extra_rc_location->first; if (FileSystemManager::IsFile(extra_rc_path)) { Configuration extra_rc_config{}; try { std::ifstream fs(extra_rc_path); auto map = Expression::FromJson(nlohmann::json::parse(fs)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "In extra RC file {}: expected an " "object but found:\n{}", extra_rc_path.string(), map->ToString()); std::exit(kExitConfigError); } extra_rc_config = Configuration{map}; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing extra RC file {} as JSON failed " "with error:\n{}", extra_rc_path.string(), e.what()); std::exit(kExitConfigError); } rc_config = MergeMRRC(rc_config, extra_rc_config); } } } } // If requested, dump effective rc if (clargs->common.dump_rc) { auto dump_json = rc_config.ToJson(); dump_json.erase("rc files"); std::ofstream os(*clargs->common.dump_rc); os << dump_json.dump(2) << std::endl; } // read local build root; overwritten if user provided it already if (not clargs->common.just_mr_paths->root) { auto build_root = ReadLocation(rc_config["local build root"], clargs->common.just_mr_paths->workspace_root); if (build_root) { clargs->common.just_mr_paths->root = build_root->first; } } // read checkout locations file; overwritten if user provided it already if (not clargs->common.checkout_locations_file) { auto checkout = ReadLocation(rc_config["checkout locations"], clargs->common.just_mr_paths->workspace_root); if (checkout) { if (not FileSystemManager::IsFile(checkout->first)) { Logger::Log(LogLevel::Error, "Cannot find checkout locations file {}.", checkout->first.string()); std::exit(kExitConfigError); } clargs->common.checkout_locations_file = checkout->first; } } // read distdirs; user can append, but does not overwrite auto distdirs = rc_config["distdirs"]; if (distdirs.IsNotNull()) { if (not distdirs->IsList()) { Logger::Log(LogLevel::Error, "Configuration-file provided distdirs has to be a list " "of strings, but found {}", distdirs->ToString()); std::exit(kExitConfigError); } auto const& distdirs_list = distdirs->List(); for (auto const& l : distdirs_list) { auto paths = ReadLocation(l, clargs->common.just_mr_paths->workspace_root); if (paths) { if (FileSystemManager::IsDirectory(paths->first)) { clargs->common.just_mr_paths->distdirs.emplace_back( paths->first); } else { Logger::Log(LogLevel::Warning, "Ignoring non-existing distdir {}.", paths->first.string()); } } } } // read just path; overwritten if user provided it already if (not clargs->common.just_path) { auto just = ReadLocation(rc_config["just"], clargs->common.just_mr_paths->workspace_root); if (just) { clargs->common.just_path = just->first; } } // read git binary path; overwritten if user provided it already if (not clargs->common.git_path) { auto git = ReadLocation(rc_config["git"], clargs->common.just_mr_paths->workspace_root); if (git) { clargs->common.git_path = git->first; } } // read the just file-arguments auto just_files = rc_config["just files"]; if (just_files.IsNotNull()) { if (not just_files->IsMap()) { Logger::Log(LogLevel::Error, "Configuration-file provided 'just files' has to be a " "map, but found {}.", just_files->ToString()); std::exit(kExitConfigError); } auto files = Configuration(just_files); clargs->just_cmd.config = ReadOptionalLocationList( files["config"], clargs->common.just_mr_paths->workspace_root, "'config' in 'just files'"); clargs->just_cmd.endpoint_configuration = ReadOptionalLocationList( files["endpoint-configuration"], clargs->common.just_mr_paths->workspace_root, "'endpoint-configuration' in 'just files'"); } // read additional just args; user can append, but does not overwrite auto just_args = rc_config["just args"]; if (just_args.IsNotNull()) { if (not just_args->IsMap()) { Logger::Log(LogLevel::Error, "Configuration-file provided 'just' arguments has to " "be a map, but found {}", just_args->ToString()); std::exit(kExitConfigError); } for (auto const& [cmd_name, cmd_args] : just_args->Map()) { // get list of string args for current command std::vector args{}; if (not cmd_args->IsList()) { Logger::Log( LogLevel::Error, "Configuration-file provided 'just' argument key {} has to " "have as value a list of strings, but found {}", cmd_name, cmd_args->ToString()); std::exit(kExitConfigError); } auto const& args_list = cmd_args->List(); args.reserve(args_list.size()); for (auto const& arg : args_list) { if (not arg->IsString()) { Logger::Log( LogLevel::Error, "Configuration-file provided 'just' argument key {} " "must have strings in its list value, but found {}", cmd_name, arg->ToString()); std::exit(kExitConfigError); } args.emplace_back(arg->String()); } clargs->just_cmd.just_args[cmd_name] = std::move(args); } } // read remote-execution properties, used for extending the launch // command-line (not settable on just-mr's command line). auto re_props = rc_config["remote-execution properties"]; if (re_props.IsNotNull()) { if (not re_props->IsList()) { Logger::Log(LogLevel::Error, "Configuration-file provided remote-execution " "properties have to be a list of strings, but found {}", re_props->ToString()); std::exit(kExitConfigError); } for (auto const& entry : re_props->List()) { if (not entry->IsString()) { Logger::Log( LogLevel::Error, "Configuration-file provided remote-execution properties " "have to be a list of strings, but found entry {}", entry->ToString()); std::exit(kExitConfigError); } clargs->launch_fwd.remote_execution_properties.emplace_back( entry->String()); } } // read the defaults for the retry parameters if (not clargs->retry.max_attempts) { auto max_attempts = rc_config["max attempts"]; if (max_attempts.IsNotNull()) { if (not max_attempts->IsNumber()) { Logger::Log(LogLevel::Error, "Configuration-file provided \"max attempts\" has " "to be a number, but found {}", max_attempts->ToString()); std::exit(kExitConfigError); } clargs->retry.max_attempts = static_cast(std::lround(max_attempts->Number())); } } if (not clargs->retry.initial_backoff_seconds) { auto initial_backoff_seconds = rc_config["initial backoff seconds"]; if (initial_backoff_seconds.IsNotNull()) { if (not initial_backoff_seconds->IsNumber()) { Logger::Log(LogLevel::Error, "Configuration-file provided \"initial backoff " "seconds\" has to be a number, but found {}", initial_backoff_seconds->ToString()); std::exit(kExitConfigError); } clargs->retry.initial_backoff_seconds = static_cast( std::lround(initial_backoff_seconds->Number())); } } if (not clargs->retry.max_backoff_seconds) { auto max_backoff_seconds = rc_config["max backoff seconds"]; if (max_backoff_seconds.IsNotNull()) { if (not max_backoff_seconds->IsNumber()) { Logger::Log(LogLevel::Error, "Configuration-file provided \"max backoff " "seconds\" has to be a number, but found {}", max_backoff_seconds->ToString()); std::exit(kExitConfigError); } clargs->retry.max_backoff_seconds = static_cast( std::lround(max_backoff_seconds->Number())); } } // read default for local launcher if (not clargs->common.local_launcher) { auto launcher = rc_config["local launcher"]; if (launcher.IsNotNull()) { if (not launcher->IsList()) { Logger::Log(LogLevel::Error, "Configuration-file provided launcher has to be a " "list of strings, but found {}", launcher->ToString()); std::exit(kExitConfigError); } std::vector default_launcher{}; default_launcher.reserve(launcher->List().size()); for (auto const& entry : launcher->List()) { if (not entry->IsString()) { Logger::Log(LogLevel::Error, "Configuration-file provided launcher {} is " "not a list of strings", launcher->ToString()); std::exit(kExitConfigError); } default_launcher.emplace_back(entry->String()); } clargs->common.local_launcher = default_launcher; } else { clargs->common.local_launcher = kDefaultLauncher; } } // Set log limit, if specified and not set on the command line if (not clargs->log.log_limit) { auto limit = rc_config["log limit"]; if (limit.IsNotNull()) { if (not limit->IsNumber()) { Logger::Log(LogLevel::Error, "Configuration-file specified log-limit has to be " "a number, but found {}", limit->ToString()); std::exit(kExitConfigError); } clargs->log.log_limit = ToLogLevel(limit->Number()); } } // Set restrict stderr log limit, if specified and not set on the command // line if (not clargs->log.restrict_stderr_log_limit) { auto limit = rc_config["restrict stderr log limit"]; if (limit.IsNotNull()) { if (not limit->IsNumber()) { Logger::Log(LogLevel::Error, "Configuration-file specified log-limit has to be " "a number, but found {}", limit->ToString()); std::exit(kExitConfigError); } clargs->log.restrict_stderr_log_limit = ToLogLevel(limit->Number()); } } // Add additional log sinks specified in the rc file. auto log_files = rc_config["log files"]; if (log_files.IsNotNull()) { if (not log_files->IsList()) { Logger::Log(LogLevel::Error, "Configuration-provided log files have to be a list of " "location objects, but found {}", log_files->ToString()); std::exit(kExitConfigError); } auto const& files_list = log_files->List(); clargs->log.log_files.reserve(clargs->log.log_files.size() + files_list.size()); for (auto const& log_file : files_list) { auto path = ReadLocation( log_file, clargs->common.just_mr_paths->workspace_root); if (path) { clargs->log.log_files.emplace_back(path->first); } } } // read remote exec args; overwritten if user provided it already auto remote = rc_config["remote execution"]; if (remote.IsNotNull()) { if (not remote->IsMap()) { Logger::Log(LogLevel::Error, "Configuration-provided remote execution arguments has " "to be a map, but found {}", remote->ToString()); std::exit(kExitConfigError); } if (not clargs->common.remote_execution_address) { auto addr = remote->Get("address", Expression::none_t{}); if (addr.IsNotNull()) { if (not addr->IsString()) { Logger::Log(LogLevel::Error, "Configuration-provided remote execution " "address has to be a string, but found {}", addr->ToString()); std::exit(kExitConfigError); } clargs->common.remote_execution_address = addr->String(); } } if (not clargs->common.remote_instance_name) { auto name = remote->Get("instance name", Expression::none_t{}); if (name.IsNotNull()) { if (not name->IsString()) { Logger::Log(LogLevel::Error, "Configuration-provided remote instance name " "has to be a string, but found {}", name->ToString()); std::exit(kExitConfigError); } clargs->common.remote_instance_name = name->String(); } } if (not clargs->common.compatible) { auto compat = remote->Get("compatible", Expression::none_t{}); if (compat.IsNotNull()) { if (not compat->IsBool()) { Logger::Log(LogLevel::Error, "Configuration-provided remote execution " "compatibility has to be a flag, but found {}", compat->ToString()); std::exit(kExitConfigError); } clargs->common.compatible = compat->Bool(); } } } // read remote exec args; overwritten if user provided it already auto serve = rc_config["remote serve"]; if (serve.IsNotNull()) { if (not serve->IsMap()) { Logger::Log(LogLevel::Error, "Configuration-provided remote serve service arguments " "has to be a map, but found {}", serve->ToString()); std::exit(kExitConfigError); } if (not clargs->common.remote_serve_address) { auto addr = serve->Get("address", Expression::none_t{}); if (addr.IsNotNull()) { if (not addr->IsString()) { Logger::Log(LogLevel::Error, "Configuration-provided remote serve service " "address has to be a string, but found {}", addr->ToString()); std::exit(kExitConfigError); } clargs->common.remote_serve_address = addr->String(); } } } // read authentication args; overwritten if user provided it already auto auth_args = rc_config["authentication"]; if (auth_args.IsNotNull()) { if (not auth_args->IsMap()) { Logger::Log(LogLevel::Error, "Configuration-provided authentication arguments has " "to be a map, but found {}", auth_args->ToString()); std::exit(kExitConfigError); } if (not clargs->auth.tls_ca_cert) { auto v = ReadLocation(auth_args->Get("ca cert", Expression::none_t{}), clargs->common.just_mr_paths->workspace_root); if (v) { clargs->auth.tls_ca_cert = v->first; } } if (not clargs->auth.tls_client_cert) { auto v = ReadLocation( auth_args->Get("client cert", Expression::none_t{}), clargs->common.just_mr_paths->workspace_root); if (v) { clargs->auth.tls_client_cert = v->first; } } if (not clargs->auth.tls_client_key) { auto v = ReadLocation(auth_args->Get("client key", Expression::none_t{}), clargs->common.just_mr_paths->workspace_root); if (v) { clargs->auth.tls_client_key = v->first; } } } // read path to absent repository specification if not already provided by // the user if (not clargs->common.absent_repository_file) { auto absent_order = rc_config["absent"]; if (absent_order.IsNotNull() and absent_order->IsList()) { for (auto const& entry : absent_order->List()) { auto path = ReadLocation( entry, clargs->common.just_mr_paths->workspace_root); if (path and FileSystemManager::IsFile(path->first)) { clargs->common.absent_repository_file = path->first; break; } } } } // read invocation log argumnets auto invocation_log = rc_config["invocation log"]; if (invocation_log.IsNotNull()) { if (not invocation_log->IsMap()) { Logger::Log( LogLevel::Error, "Value of \"invocation log\" has to be a map, but found {}", invocation_log->ToString()); std::exit(kExitConfigError); } auto dir = ReadLocation(invocation_log->Get("directory", Expression::none_t{}), clargs->common.just_mr_paths->workspace_root); if (dir) { clargs->invocation_log.directory = dir->first; // Parse the remaining entries, only if directory is specified auto proj_id = invocation_log->Get("project id", Expression::none_t{}); if (proj_id->IsString()) { clargs->invocation_log.project_id = proj_id->String(); } auto metadata = invocation_log->Get("metadata", Expression::none_t{}); if (metadata->IsString()) { clargs->invocation_log.metadata = metadata->String(); } auto graph_file = invocation_log->Get("--dump-graph", Expression::none_t{}); if (graph_file->IsString()) { clargs->invocation_log.graph_file = graph_file->String(); } auto graph_file_plain = invocation_log->Get("--dump-plain-graph", Expression::none_t{}); if (graph_file_plain->IsString()) { clargs->invocation_log.graph_file_plain = graph_file_plain->String(); } auto dump_artifacts_to_build = invocation_log->Get( "--dump-artifacts-to-build", Expression::none_t{}); if (dump_artifacts_to_build->IsString()) { clargs->invocation_log.dump_artifacts_to_build = dump_artifacts_to_build->String(); } auto dump_artifacts = invocation_log->Get("--dump-artifacts", Expression::none_t{}); if (dump_artifacts->IsString()) { clargs->invocation_log.dump_artifacts = dump_artifacts->String(); } auto profile = invocation_log->Get("--profile", Expression::none_t{}); if (profile->IsString()) { clargs->invocation_log.profile = profile->String(); } auto msg = invocation_log->Get("invocation message", Expression::none_t{}); if (msg->IsString()) { clargs->invocation_log.invocation_msg = msg->String(); } auto context_vars = invocation_log->Get("context variables", Expression::none_t{}); if (context_vars->IsList()) { for (auto const& env_var : context_vars->List()) { if (env_var->IsString()) { clargs->invocation_log.context_vars.emplace_back( env_var->String()); } } } } } // read config lookup order auto config_lookup_order = rc_config["config lookup order"]; if (config_lookup_order.IsNotNull()) { for (auto const& entry : config_lookup_order->List()) { auto paths = ReadLocation( entry, clargs->common.just_mr_paths->workspace_root); if (paths and FileSystemManager::IsFile(paths->first)) { clargs->common.just_mr_paths->setup_root = paths->second; return paths->first; } } } else { for (auto const& entry : kDefaultConfigLocations) { auto paths = ReadLocation( entry, clargs->common.just_mr_paths->workspace_root); if (paths and FileSystemManager::IsFile(paths->first)) { clargs->common.just_mr_paths->setup_root = paths->second; return paths->first; } } } return std::nullopt; // default return value } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/rc.hpp000066400000000000000000000021631516554100600255510ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_HPP #include #include #include "gsl/gsl" #include "src/other_tools/just_mr/cli.hpp" /// \brief Read just-mrrc file and set up various configs. Return the path to /// the repository config file, if any is provided. [[nodiscard]] auto ReadJustMRRC( gsl::not_null const& clargs) -> std::optional; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/rc_merge.cpp000066400000000000000000000046701516554100600267300ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/rc_merge.hpp" #include #include #include #include #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" namespace { auto const kAccumulating = std::vector{"distdirs"}; auto const kLocalMerge = std::vector{"just args", "just files", "invocation log"}; } // namespace [[nodiscard]] auto MergeMRRC(const Configuration& base, const Configuration& delta) noexcept -> Configuration { // For most fields, just let the delta entry override auto result = base.Update(delta.Expr()); // Accumulating fields for (auto const& f : kAccumulating) { auto joined = Expression::list_t{}; auto const& deltaf = delta[f]; if (deltaf->IsList()) { std::copy(deltaf->List().begin(), deltaf->List().end(), std::back_inserter(joined)); } auto const& basef = base[f]; if (basef->IsList()) { std::copy(basef->List().begin(), basef->List().end(), std::back_inserter(joined)); } result = result.Update( ExpressionPtr{Expression::map_t{f, ExpressionPtr{joined}}}); } // Locally-merging fields for (auto const& f : kLocalMerge) { auto joined = Configuration{Expression::kEmptyMap}; auto const& basef = base[f]; if (basef->IsMap()) { joined = joined.Update(basef); } auto const& deltaf = delta[f]; if (deltaf->IsMap()) { joined = joined.Update(deltaf); } result = result.Update(ExpressionPtr{Expression::map_t{f, joined.Expr()}}); } return result; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/rc_merge.hpp000066400000000000000000000020501516554100600267230ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_MERGE_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_MERGE_HPP #include "src/buildtool/build_engine/expression/configuration.hpp" /// \brief Given two rc values combine in a latest-wins fashion. [[nodiscard]] auto MergeMRRC(const Configuration& base, const Configuration& delta) noexcept -> Configuration; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_RC_MERGE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/setup.cpp000066400000000000000000000574351516554100600263140ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/setup.hpp" #include #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/retry.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/other_tools/repo_map/repos_to_setup_map.hpp" #include "src/other_tools/root_maps/commit_git_map.hpp" #include "src/other_tools/root_maps/content_git_map.hpp" #include "src/other_tools/root_maps/distdir_git_map.hpp" #include "src/other_tools/root_maps/foreign_file_git_map.hpp" #include "src/other_tools/root_maps/fpath_git_map.hpp" #include "src/other_tools/root_maps/tree_id_git_map.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/file_locking.hpp" auto MultiRepoSetup(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, StorageConfig const& native_storage_config, Storage const& native_storage, bool interactive, std::string const& multi_repo_tool_name) -> std::optional> { // provide report Logger::Log(LogLevel::Info, "Performing repositories setup"); // set anchor dir to setup_root; current dir will be reverted when anchor // goes out of scope auto cwd_anchor = FileSystemManager::ChangeDirectory( common_args.just_mr_paths->setup_root); auto repos = (*config)["repositories"]; if (not repos.IsNotNull()) { Logger::Log(LogLevel::Error, "Config: Mandatory key \"repositories\" missing"); return std::nullopt; } if (not repos->IsMap()) { Logger::Log(LogLevel::Error, "Config: Value for key \"repositories\" is not a map"); return std::nullopt; } auto setup_repos = std::make_shared(); // repos to setup and include nlohmann::json mr_config{}; // output config to populate auto main = common_args.main; // get local copy of updated clarg 'main', as // it might be updated again from config // check if config provides main repo name if (not main) { auto main_from_config = (*config)["main"]; if (main_from_config.IsNotNull()) { if (main_from_config->IsString()) { main = main_from_config->String(); } else { Logger::Log( LogLevel::Error, "Unsupported value {} for field \"main\" in configuration.", main_from_config->ToString()); return std::nullopt; } } } // pass on main that was explicitly set via command line or config if (main) { mr_config["main"] = *main; } // get default repos to setup and to include JustMR::Utils::DefaultReachableRepositories(repos, setup_repos); // check if main is to be taken as first repo name lexicographically if (not main and not setup_repos->to_setup.empty()) { main = *std::min_element(setup_repos->to_setup.begin(), setup_repos->to_setup.end()); } // final check on which repos are to be set up if (main and not setup_args.sub_all) { JustMR::Utils::ReachableRepositories(repos, *main, setup_repos); } Logger::Log(LogLevel::Info, "Found {} repositories involved", setup_repos->to_setup.size()); // setup local execution config auto const local_exec_config = JustMR::Utils::CreateLocalExecutionConfig(common_args); if (not local_exec_config) { return std::nullopt; } // pack the native local context and create api LocalContext const native_local_context{ .exec_config = &*local_exec_config, .storage_config = &native_storage_config, .storage = &native_storage}; IExecutionApi::Ptr const native_local_api = std::make_shared(&native_local_context); // pack the compatible local context, if needed std::unique_ptr compat_storage_config = nullptr; std::unique_ptr compat_storage = nullptr; std::unique_ptr compat_local_context = nullptr; std::optional compat_lock = std::nullopt; IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder::Rebuild(native_storage_config) .SetHashType(HashFunction::Type::PlainSHA256) .Build(); if (not config) { Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } compat_storage_config = std::make_unique(*std::move(config)); compat_storage = std::make_unique( Storage::Create(compat_storage_config.get())); compat_local_context = std::make_unique( LocalContext{.exec_config = &*local_exec_config, .storage_config = compat_storage_config.get(), .storage = compat_storage.get()}); // if a compatible storage is created, one must get a lock for it the // same way as done for the native one compat_lock = GarbageCollector::SharedLock(*compat_storage_config); if (not compat_lock) { Logger::Log(LogLevel::Error, "Failed to acquire compatible storage gc lock"); return std::nullopt; } compat_local_api = std::make_shared(&*compat_local_context); } // setup the overall local api, aware of compatibility IExecutionApi::Ptr mr_local_api = std::make_shared( &native_local_context, &*native_local_api, common_args.compatible ? &*compat_local_context : nullptr, common_args.compatible ? &*compat_local_api : nullptr); // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { return std::nullopt; } // setup the retry config auto const retry_config = CreateRetryConfig(retry_args); if (not retry_config) { return std::nullopt; } // setup remote execution config auto const remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig( common_args.remote_execution_address, common_args.remote_serve_address, common_args.remote_instance_name ? *common_args.remote_instance_name : std::string{}); if (not remote_exec_config) { return std::nullopt; } // create the remote api IExecutionApi::Ptr remote_api = nullptr; if (auto const address = remote_exec_config->remote_address) { auto const hash_fct = compat_local_context != nullptr ? compat_local_context->storage_config->hash_function : native_local_context.storage_config->hash_function; ExecutionConfiguration config; config.skip_cache_lookup = false; remote_api = std::make_shared(remote_exec_config->remote_instance_name, address->host, address->port, &*auth_config, &*retry_config, config, hash_fct, mr_local_api->GetTempSpace()); } bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, .retry_config = &*retry_config, .exec_config = &*remote_exec_config}; // setup the api for serving roots auto serve_config = JustMR::Utils::CreateServeConfig(common_args.remote_serve_address); if (not serve_config) { return std::nullopt; } auto const apis = ApiBundle{.local = mr_local_api, .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr ? &*compat_local_context : &native_local_context, // defines the client's hash_function &remote_context, &apis /*unused*/); // check configuration of the serve endpoint provided if (serve) { // if we have a remote endpoint explicitly given by the user, it must // match what the serve endpoint expects if (common_args.remote_execution_address and not serve->CheckServeRemoteExecution()) { return std::nullopt; // this check logs error on failure } // check the compatibility mode of the serve endpoint auto compatible = serve->IsCompatible(); if (not compatible) { Logger::Log(LogLevel::Warning, "Checking compatibility configuration of the provided " "serve endpoint failed."); serve = std::nullopt; } if (*compatible != common_args.compatible) { Logger::Log( LogLevel::Warning, "Provided serve endpoint operates in a different compatibility " "mode than stated. Serve endpoint ignored."); serve = std::nullopt; } } // setup progress and statistics instances JustMRStatistics stats{}; JustMRProgress progress{setup_repos->to_setup.size()}; // setup the required async maps auto crit_git_op_ptr = std::make_shared(); auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr); auto content_cas_map = CreateContentCASMap( common_args.just_mr_paths, common_args.alternative_mirrors, common_args.ca_info, &critical_git_op_map, serve ? &*serve : nullptr, &native_storage_config, compat_storage_config != nullptr ? &*compat_storage_config : nullptr, &native_storage, compat_storage != nullptr ? &*compat_storage : nullptr, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, &progress, common_args.jobs); auto import_to_git_map = CreateImportToGitMap(&critical_git_op_map, common_args.git_path->string(), *common_args.local_launcher, &native_storage_config, common_args.jobs); auto git_tree_fetch_map = CreateGitTreeFetchMap( &critical_git_op_map, &import_to_git_map, common_args.git_path->string(), *common_args.local_launcher, common_args.alternative_mirrors, serve ? &*serve : nullptr, &native_storage_config, compat_storage_config != nullptr ? &*compat_storage_config : nullptr, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, false, /* backup_to_remote */ &progress, common_args.jobs); auto resolve_symlinks_map = CreateResolveSymlinksMap(); auto commit_git_map = CreateCommitGitMap(&critical_git_op_map, &import_to_git_map, common_args.just_mr_paths, common_args.alternative_mirrors, common_args.git_path->string(), *common_args.local_launcher, serve ? &*serve : nullptr, &native_storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.fetch_absent, &progress, common_args.jobs); auto content_git_map = CreateContentGitMap(&content_cas_map, &import_to_git_map, common_args.just_mr_paths, &resolve_symlinks_map, &critical_git_op_map, serve ? &*serve : nullptr, &native_storage_config, &native_storage, common_args.fetch_absent, &progress, common_args.jobs); auto foreign_file_git_map = CreateForeignFileGitMap(&content_cas_map, &import_to_git_map, serve ? &*serve : nullptr, &native_storage_config, &native_storage, common_args.fetch_absent, common_args.jobs); auto fpath_git_map = CreateFilePathGitMap( just_cmd_args.subcmd_name, &critical_git_op_map, &import_to_git_map, &resolve_symlinks_map, serve ? &*serve : nullptr, &native_storage_config, common_args.jobs, multi_repo_tool_name, common_args.just_path ? common_args.just_path->string() : kDefaultJustPath); auto distdir_git_map = CreateDistdirGitMap(&content_cas_map, &import_to_git_map, &critical_git_op_map, serve ? &*serve : nullptr, &native_storage_config, &native_storage, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); auto tree_id_git_map = CreateTreeIdGitMap(&git_tree_fetch_map, &critical_git_op_map, &import_to_git_map, common_args.fetch_absent, serve ? &*serve : nullptr, &native_storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); auto repos_to_setup_map = CreateReposToSetupMap(config, main, interactive, &commit_git_map, &content_git_map, &foreign_file_git_map, &fpath_git_map, &distdir_git_map, &tree_id_git_map, common_args.fetch_absent, &stats, common_args.jobs); // set up progress observer std::atomic done{false}; std::condition_variable cv{}; auto reporter = JustMRProgressReporter::Reporter(&stats, &progress); auto observer = std::thread([reporter, &done, &cv]() { reporter(&done, &cv); }); // Populate workspace_root and TAKE_OVER fields bool failed{false}; bool has_value{false}; { TaskSystem ts{common_args.jobs}; repos_to_setup_map.ConsumeAfterKeysReady( &ts, setup_repos->to_setup, [&failed, &has_value, &mr_config, repos, setup_repos, main, interactive, multi_repo_tool_name](auto const& values) noexcept { try { has_value = true; // set the initial setup repositories nlohmann::json mr_repos{}; for (auto const& repo : setup_repos->to_setup) { auto i = static_cast( &repo - setup_repos->to_setup.data()); // get index mr_repos[repo] = *values[i]; } // populate ALT_DIRS constexpr auto kErrMsgFormat = "While performing {} {}:\nWhile populating fields for " "repository {}:\nExpected value for key \"{}\" to be a " "string, but found {}"; for (auto const& repo : setup_repos->to_include) { auto desc = repos->Get(repo, Expression::none_t{}); if (desc.IsNotNull()) { if (not((main and (repo == *main)) and interactive)) { for (auto const& key : kAltDirs) { auto val = desc->Get(key, Expression::none_t{}); if (val.IsNotNull()) { // we expect a string if (not val->IsString()) { Logger::Log(LogLevel::Error, kErrMsgFormat, multi_repo_tool_name, interactive ? "setup-env" : "setup", repo, key, val->ToString()); failed = true; return; } if (not((main and (val->String() == *main)) and interactive)) { mr_repos[repo][key] = mr_repos[val->String()] ["workspace_root"]; } // otherwise, continue } } } } } // retain only the repos we need for (auto const& repo : setup_repos->to_include) { mr_config["repositories"][repo] = mr_repos[repo]; } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "While performing {} {}:\nPopulating the " "configuration failed with:\n{}", multi_repo_tool_name, interactive ? "setup-env" : "setup", ex.what()); failed = true; } }, [&failed, interactive, multi_repo_tool_name](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While performing {} {}:\n{}", multi_repo_tool_name, interactive ? "setup-env" : "setup", msg); failed = failed or fatal; }); } // close progress observer done = true; cv.notify_all(); observer.join(); if (failed) { return std::nullopt; } if (not has_value) { // check for cycles in maps where cycles can occur and have meaning if (auto error = DetectAndReportCycle("resolving symlinks", resolve_symlinks_map, kGitObjectToResolvePrinter)) { Logger::Log(LogLevel::Error, *error); return std::nullopt; } DetectAndReportPending( "setup", repos_to_setup_map, kReposToSetupPrinter); return std::nullopt; } // if successful, return the output config auto const& cas = native_storage.CAS(); auto digest = cas.StoreBlob(mr_config.dump(2)); if (not digest) { return std::nullopt; } auto blob_path = cas.BlobPath(*digest, /*is_executable=*/false); if (not blob_path) { return std::nullopt; } return std::make_optional(std::make_pair(*blob_path, digest->hash())); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/setup.hpp000066400000000000000000000034171516554100600263100ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_HPP #include #include #include #include #include #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/retry_cli.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Setup for a multi-repository build. Return the pair /// of path and hash (as hex-string) of the multi-repository configuration. [[nodiscard]] auto MultiRepoSetup( std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, StorageConfig const& native_storage_config, Storage const& native_storage, bool interactive, std::string const& multi_repo_tool_name) -> std::optional>; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/setup_utils.cpp000066400000000000000000000355251516554100600275300ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/setup_utils.hpp" #include #include #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/utils/parse_precomputed_root.hpp" #include "src/utils/cpp/expected.hpp" namespace { void WarnUnknownKeys(std::string const& name, ExpressionPtr const& repo_def) { if (not repo_def->IsMap()) { return; } for (auto const& [key, value] : repo_def->Map()) { if (not kRepositoryExpectedFields.contains(key)) { Logger::Log(std::any_of(kRepositoryPossibleFieldTrunks.begin(), kRepositoryPossibleFieldTrunks.end(), [k = key](auto const& trunk) { return k.find(trunk) != std::string::npos; }) ? LogLevel::Debug : LogLevel::Warning, "Ignoring unknown field {} in repository {}", key, name); } } } [[nodiscard]] auto GetTargetRepoIfPrecomputed(ExpressionPtr const& repos, std::string const& name) -> std::optional { // Resolve indirections while the root's workspace root is declared // implicitly: ExpressionPtr root{name}; while (root.IsNotNull() and root->IsString()) { auto const repo = repos->Get(root->String(), Expression::none_t{}); if (not repo.IsNotNull() or not repo->IsMap()) { return std::nullopt; } root = repo->Get("repository", Expression::none_t{}); } // Check the root is a precomputed root: if (auto const precomputed = ParsePrecomputedRoot(root)) { return precomputed->GetReferencedRepository(); } return std::nullopt; } [[nodiscard]] auto IsAbsent(ExpressionPtr const& repo_def) -> bool { if (repo_def.IsNotNull() and repo_def->IsMap()) { if (auto repo = repo_def->Get("repository", Expression::none_t{}); repo.IsNotNull() and repo->IsMap()) { if (auto pragma = repo->Get("pragma", Expression::none_t{}); pragma.IsNotNull() and pragma->IsMap()) { auto absent = pragma->Get("absent", Expression::none_t{}); return absent.IsNotNull() and absent->IsBool() and absent->Bool(); } } } return false; } [[nodiscard]] auto IsNotContentFixed(ExpressionPtr const& repo_def) -> bool { if (not repo_def.IsNotNull() or not repo_def->IsMap()) { return false; } if (auto repo = repo_def->Get("repository", Expression::none_t{}); repo.IsNotNull() and repo->IsMap()) { // Check if type == "file" auto type = repo->Get("type", Expression::none_t{}); if (not type.IsNotNull() or not type->IsString()) { return false; } if (type->String() == "file") { auto pragma = repo->Get("pragma", Expression::none_t{}); if (not pragma.IsNotNull() or not pragma->IsMap()) { return true; // not content-fixed if not to_git } // Check for explicit to_git == true if (auto to_git = pragma->Get("to_git", Expression::none_t{}); to_git.IsNotNull() and to_git->IsBool() and to_git->Bool()) { return false; } // Check for implicit to_git == true if (auto special = pragma->Get("special", Expression::none_t{}); special.IsNotNull() and special->IsString()) { auto const& special_str = special->String(); if (special_str == "resolve-partially" or special_str == "resolve-completely") { return false; } } return true; // not content-fixed if not to_git } } return false; } } // namespace namespace JustMR::Utils { void ReachableRepositories( ExpressionPtr const& repos, std::string const& main, std::shared_ptr const& setup_repos) { // use temporary sets to avoid duplicates std::unordered_set include_repos_set; std::unordered_set setup_repos_set; bool absent_main = IsAbsent(repos->Get(main, Expression::none_t{})); // traverse all bindings of main repository for (std::queue to_process({main}); not to_process.empty(); to_process.pop()) { auto const& repo_name = to_process.front(); // Check the repo hasn't been processed yet if (not include_repos_set.insert(repo_name).second) { continue; } auto const repos_repo_name = repos->Get(repo_name, Expression::none_t{}); if (not repos_repo_name.IsNotNull()) { continue; } WarnUnknownKeys(repo_name, repos_repo_name); // Warn if main repo is marked absent and current repo (including main) // is not content-fixed if (absent_main and IsNotContentFixed(repos_repo_name)) { Logger::Log(LogLevel::Warning, "Found non-content-fixed repository {} as dependency " "of absent main repository {}", nlohmann::json(repo_name).dump(), nlohmann::json(main).dump()); } // If the current repo is a computed one, process its target repo if (auto precomputed = GetTargetRepoIfPrecomputed(repos, repo_name)) { to_process.push(*std::move(precomputed)); } // check bindings auto const bindings = repos_repo_name->Get("bindings", Expression::none_t{}); if (bindings.IsNotNull() and bindings->IsMap()) { for (auto const& bound : bindings->Map().Values()) { if (bound.IsNotNull() and bound->IsString()) { to_process.push(bound->String()); } } } for (auto const& layer : kAltDirs) { auto const layer_val = repos_repo_name->Get(layer, Expression::none_t{}); if (layer_val.IsNotNull() and layer_val->IsString()) { auto const layer_repo_name = layer_val->String(); setup_repos_set.insert(layer_repo_name); // If the overlay repo is a computed one, process its target // repo if (auto precomputed = GetTargetRepoIfPrecomputed(repos, layer_repo_name)) { to_process.push(*std::move(precomputed)); } } } } setup_repos_set.insert(include_repos_set.begin(), include_repos_set.end()); // copy to vectors setup_repos->to_setup.clear(); setup_repos->to_setup.reserve(setup_repos_set.size()); std::copy( setup_repos_set.begin(), setup_repos_set.end(), std::inserter(setup_repos->to_setup, setup_repos->to_setup.end())); setup_repos->to_include.clear(); setup_repos->to_include.reserve(include_repos_set.size()); std::copy( include_repos_set.begin(), include_repos_set.end(), std::inserter(setup_repos->to_include, setup_repos->to_include.end())); } void DefaultReachableRepositories( ExpressionPtr const& repos, std::shared_ptr const& setup_repos) { setup_repos->to_setup = repos->Map().Keys(); setup_repos->to_include = setup_repos->to_setup; } auto ReadConfiguration( std::optional const& config_file_opt, std::optional const& absent_file_opt) -> std::shared_ptr { if (not config_file_opt) { Logger::Log(LogLevel::Error, "Cannot find repository configuration."); std::exit(kExitConfigError); } auto const& config_file = *config_file_opt; auto config = nlohmann::json::object(); if (not FileSystemManager::IsFile(config_file)) { Logger::Log(LogLevel::Error, "Cannot read config file {}.", config_file.string()); std::exit(kExitConfigError); } try { std::ifstream fs(config_file); config = nlohmann::json::parse(fs); if (not config.is_object()) { Logger::Log(LogLevel::Error, "Config file {} does not contain a JSON object.", config_file.string()); std::exit(kExitConfigError); } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing config file {} failed with error:\n{}", config_file.string(), e.what()); std::exit(kExitConfigError); } if (absent_file_opt) { if (not FileSystemManager::IsFile(*absent_file_opt)) { Logger::Log(LogLevel::Error, "Not file specifying the absent repositories: {}", absent_file_opt->string()); std::exit(kExitConfigError); } try { std::ifstream fs(*absent_file_opt); auto absent = nlohmann::json::parse(fs); if (not absent.is_array()) { Logger::Log(LogLevel::Error, "Expected {} to contain a list of repository " "names, but found {}", absent_file_opt->string(), absent.dump()); std::exit(kExitConfigError); } std::unordered_set absent_set{}; for (auto const& repo : absent) { if (not repo.is_string()) { Logger::Log(LogLevel::Error, "Repositories names have to be strings, but " "found entry {} in {}", repo.dump(), absent_file_opt->string()); std::exit(kExitConfigError); } absent_set.insert(repo.get()); } auto new_repos = nlohmann::json::object(); auto repos = config.value("repositories", nlohmann::json::object()); for (auto const& [key, val] : repos.items()) { new_repos[key] = val; auto ws = val.value("repository", nlohmann::json::object()); if (ws.is_object()) { auto pragma = ws.value("pragma", nlohmann::json::object()); pragma["absent"] = absent_set.contains(key); ws["pragma"] = pragma; new_repos[key]["repository"] = ws; } } config["repositories"] = new_repos; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing absent-repos file {} failed with error:\n{}", absent_file_opt->string(), e.what()); std::exit(kExitConfigError); } } try { return std::make_shared(Expression::FromJson(config)); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing configuration file failed with error:\n{}", e.what()); std::exit(kExitConfigError); } } auto CreateAuthConfig(MultiRepoRemoteAuthArguments const& authargs) noexcept -> std::optional { Auth::TLS::Builder tls_builder; tls_builder.SetCACertificate(authargs.tls_ca_cert) .SetClientCertificate(authargs.tls_client_cert) .SetClientKey(authargs.tls_client_key); // create auth config (including validation) auto result = tls_builder.Build(); if (result) { if (*result) { // correctly configured TLS/SSL certification return *std::move(*result); } Logger::Log(LogLevel::Error, result->error()); return std::nullopt; } // no TLS/SSL configuration was given, and we currently support no other // certification method, so return an empty config (no certification) return Auth{}; } auto CreateLocalExecutionConfig(MultiRepoCommonArguments const& cargs) noexcept -> std::optional { LocalExecutionConfig::Builder builder; if (cargs.local_launcher.has_value()) { builder.SetLauncher(*cargs.local_launcher); } auto config = builder.Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } auto CreateRemoteExecutionConfig( std::optional const& remote_exec_addr, std::optional const& remote_serve_addr, std::string const& remote_instance_name) noexcept -> std::optional { // if only a serve endpoint address is given, we assume it is one that acts // also as remote-execution auto remote_addr = remote_exec_addr ? remote_exec_addr : remote_serve_addr; RemoteExecutionConfig::Builder builder; auto config = builder.SetRemoteAddress(remote_addr) .SetRemoteInstanceName(remote_instance_name) .Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } auto CreateServeConfig( std::optional const& remote_serve_addr) noexcept -> std::optional { RemoteServeConfig::Builder builder; auto config = builder.SetRemoteAddress(remote_serve_addr).Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } } // namespace JustMR::Utils just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/setup_utils.hpp000066400000000000000000000102141516554100600275210ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_UTILS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_UTILS_HPP #include #include #include #include #include #include #include #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/other_tools/just_mr/cli.hpp" /* Setup-related constants and utilities for just-mr */ std::vector const kAltDirs = {"target_root", "rule_root", "expression_root"}; auto const kRepositoryExpectedFields = std::unordered_set{"bindings", "expression_file_name", "expression_root", "repository", "rule_file_name", "rule_root", "target_file_name", "target_root"}; // Substrings in repository field names that indicate commonly-used // additional keys not used by just-mr but deliberately added by the // author of the repository configuration. auto const kRepositoryPossibleFieldTrunks = std::vector{"bootstrap", "doc", "extra"}; namespace JustMR { struct SetupRepos { std::vector to_setup; std::vector to_include; }; namespace Utils { /// \brief Get the repo dependency closure for a given main repository. /// If main repository is absent, emits a warning for any reachable /// non-content-fixed repository found. /// \param repos ExpressionPtr of Map type. void ReachableRepositories( ExpressionPtr const& repos, std::string const& main, std::shared_ptr const& setup_repos); /// \brief By default, we set up and include the full repo dependency closure. /// \param repos ExpressionPtr of Map type. void DefaultReachableRepositories( ExpressionPtr const& repos, std::shared_ptr const& setup_repos); /// \brief Read in a just-mr configuration file. [[nodiscard]] auto ReadConfiguration( std::optional const& config_file_opt, std::optional const& absent_file_opt) -> std::shared_ptr; [[nodiscard]] auto CreateAuthConfig( MultiRepoRemoteAuthArguments const& authargs) noexcept -> std::optional; [[nodiscard]] auto CreateLocalExecutionConfig( MultiRepoCommonArguments const& cargs) noexcept -> std::optional; [[nodiscard]] auto CreateRemoteExecutionConfig( std::optional const& remote_exec_addr, std::optional const& remote_serve_addr, std::string const& remote_instance_name) noexcept -> std::optional; /// \brief Setup of a 'just serve' remote API based on just-mr arguments. /// \returns RemoteServeConfig if initialization was successful or std::nullopt /// if failed. [[nodiscard]] auto CreateServeConfig( std::optional const& remote_serve_addr) noexcept -> std::optional; } // namespace Utils } // namespace JustMR #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/update.cpp000066400000000000000000000324761516554100600264340ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/update.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/base_progress_reporter.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/git_update_map.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto MultiRepoUpdate(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoUpdateArguments const& update_args, StorageConfig const& native_storage_config, std::string const& multi_repo_tool_name) -> int { // provide report Logger::Log(LogLevel::Info, "Performing repositories update"); // Check trivial case if (update_args.repos_to_update.empty()) { // report success Logger::Log(LogLevel::Info, "No update needed"); // print config file std::cout << config->ToJson().dump(2) << std::endl; return kExitSuccess; } auto repos = (*config)["repositories"]; if (not repos.IsNotNull()) { Logger::Log(LogLevel::Error, "Config: Mandatory key \"repositories\" missing"); return kExitUpdateError; } if (not repos->IsMap()) { Logger::Log(LogLevel::Error, "Config: Value for key \"repositories\" is not a map"); return kExitUpdateError; } // gather repos to update std::vector repos_to_update{}; repos_to_update.reserve(update_args.repos_to_update.size()); for (auto const& repo_name : update_args.repos_to_update) { auto repo_desc_parent = repos->At(repo_name); if (not repo_desc_parent) { Logger::Log(LogLevel::Error, "Config: Missing config entry for repository {}", nlohmann::json(repo_name).dump()); return kExitUpdateError; } auto repo_desc = repo_desc_parent->get()->At("repository"); if (repo_desc) { auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc->get(), repos); if (not resolved_repo_desc) { Logger::Log(LogLevel::Error, fmt::format("Config: Found cyclic dependency for " "repository {}", nlohmann::json(repo_name).dump())); return kExitUpdateError; } if (not resolved_repo_desc.value()->IsMap()) { Logger::Log( LogLevel::Error, "Config: Repository {} resolves to a non-map description", nlohmann::json(repo_name).dump()); return kExitUpdateError; } // get repo_type auto repo_type = (*resolved_repo_desc)->At("type"); if (not repo_type) { Logger::Log( LogLevel::Error, "Config: Mandatory key \"type\" missing for repository {}", nlohmann::json(repo_name).dump()); return kExitUpdateError; } if (not repo_type->get()->IsString()) { Logger::Log(LogLevel::Error, "Config: Unsupported value {} for key \"type\" for " "repository {}", repo_type->get()->ToString(), nlohmann::json(repo_name).dump()); return kExitUpdateError; } auto repo_type_str = repo_type->get()->String(); auto const checkout_type_it = kCheckoutTypeMap.find(repo_type_str); if (checkout_type_it == kCheckoutTypeMap.end()) { Logger::Log(LogLevel::Error, "Unknown repository type {} for {}", nlohmann::json(repo_type_str).dump(), nlohmann::json(repo_name).dump()); return kExitUpdateError; } // only do work if repo is git type if (checkout_type_it->second == CheckoutType::Git) { auto repo_desc_repository = (*resolved_repo_desc)->At("repository"); if (not repo_desc_repository) { Logger::Log( LogLevel::Error, "Config: Mandatory field \"repository\" is missing"); return kExitUpdateError; } if (not repo_desc_repository->get()->IsString()) { Logger::Log(LogLevel::Error, "Config: Unsupported value {} for key " "\"repository\" for repository {}", repo_desc_repository->get()->ToString(), nlohmann::json(repo_name).dump()); return kExitUpdateError; } auto repo_desc_branch = (*resolved_repo_desc)->At("branch"); if (not repo_desc_branch) { Logger::Log( LogLevel::Error, "Config: Mandatory field \"branch\" is missing"); return kExitUpdateError; } if (not repo_desc_branch->get()->IsString()) { Logger::Log(LogLevel::Error, "Config: Unsupported value {} for key " "\"branch\" for repository {}", repo_desc_branch->get()->ToString(), nlohmann::json(repo_name).dump()); return kExitUpdateError; } std::vector inherit_env{}; auto repo_desc_inherit_env = (*resolved_repo_desc) ->Get("inherit env", Expression::kEmptyList); if (not repo_desc_inherit_env->IsList()) { Logger::Log(LogLevel::Error, "GitCheckout: optional field \"inherit env\" " "should be a list of strings, but found {}", repo_desc_inherit_env->ToString()); return kExitUpdateError; } for (auto const& var : repo_desc_inherit_env->List()) { if (not var->IsString()) { Logger::Log( LogLevel::Error, "GitCheckout: optional field \"inherit env\" " "should be a list of strings, but found entry {}", var->ToString()); return kExitUpdateError; } inherit_env.emplace_back(var->String()); } repos_to_update.emplace_back(RepoDescriptionForUpdating{ .repo = repo_desc_repository->get()->String(), .branch = repo_desc_branch->get()->String(), .inherit_env = inherit_env}); } else { Logger::Log(LogLevel::Error, "Config: Argument {} is not the name of a \"git\" " "type repository", nlohmann::json(repo_name).dump()); return kExitUpdateError; } } else { Logger::Log(LogLevel::Error, "Config: Missing repository description for {}", nlohmann::json(repo_name).dump()); return kExitUpdateError; } } // Create fake repo for the anonymous remotes auto tmp_dir = native_storage_config.CreateTypedTmpDir("update"); if (not tmp_dir) { Logger::Log(LogLevel::Error, "Failed to create commit update tmp dir"); return kExitUpdateError; } // Init and open git repo auto git_repo = GitRepoRemote::InitAndOpen(tmp_dir->GetPath(), /*is_bare=*/true); if (not git_repo) { Logger::Log(LogLevel::Error, "Failed to initialize repository in tmp dir {} for git " "commit update", tmp_dir->GetPath().string()); return kExitUpdateError; } // report progress auto nr = repos_to_update.size(); Logger::Log(LogLevel::Info, "Discovered {} Git {} to update", nr, nr == 1 ? "repository" : "repositories"); // Initialize resulting config to be updated auto mr_config = config->ToJson(); // Setup progress and statistics instances JustMRStatistics stats{}; JustMRProgress progress{repos_to_update.size()}; // Create async map auto git_update_map = CreateGitUpdateMap(git_repo->GetGitCAS(), common_args.git_path->string(), *common_args.local_launcher, common_args.alternative_mirrors, &native_storage_config, &stats, &progress, common_args.jobs); // set up progress observer std::atomic done{false}; std::condition_variable cv{}; auto reporter = JustMRProgressReporter::Reporter(&stats, &progress); auto observer = std::thread([reporter, &done, &cv]() { reporter(&done, &cv); }); // do the update bool failed{false}; bool has_value{false}; { TaskSystem ts{common_args.jobs}; git_update_map.ConsumeAfterKeysReady( &ts, repos_to_update, [&failed, &has_value, &mr_config, repos_to_update_names = update_args.repos_to_update, multi_repo_tool_name](auto const& values) noexcept { try { has_value = true; for (auto const& repo_name : repos_to_update_names) { auto i = static_cast( &repo_name - repos_to_update_names.data()); // get index // we know "repository" is a map for repo_name, so // field "commit" is here either overwritten or set if // missing; either way, this should always work mr_config["repositories"][repo_name]["repository"] ["commit"] = *values[i]; } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "While performing {} update:\nUpdating configuration " "fields failed with:\n{}", multi_repo_tool_name, ex.what()); failed = true; } }, [&failed, &multi_repo_tool_name](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, "While performing {} update:\n{}", multi_repo_tool_name, msg); failed = failed or fatal; }); } // close progress observer done = true; cv.notify_all(); observer.join(); if (failed) { return kExitUpdateError; } if (not has_value) { DetectAndReportPending( "update", git_update_map, kRepoDescriptionPrinter); return kExitUpdateError; } // report success Logger::Log(LogLevel::Info, "Update completed"); // print mr_config to stdout std::cout << mr_config.dump(2) << std::endl; return kExitSuccess; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/update.hpp000066400000000000000000000026661516554100600264370ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UPDATE_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UPDATE_HPP #include #include #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Update of Git repos commit information for a multi-repository build. [[nodiscard]] auto MultiRepoUpdate(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, MultiRepoUpdateArguments const& update_args, StorageConfig const& native_storage_config, std::string const& multi_repo_tool_name) -> int; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UPDATE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/utils.cpp000066400000000000000000000047341516554100600263060ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/utils.hpp" #include #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" namespace JustMR::Utils { auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos, gsl::not_null*> const& seen) -> std::optional { if (not repo_desc->IsString()) { return repo_desc; } auto desc_str = repo_desc->String(); if (seen->contains(desc_str)) { // cyclic dependency return std::nullopt; } [[maybe_unused]] auto insert_res = seen->insert(desc_str); auto const& new_repo_desc = repos[desc_str]; if (not new_repo_desc->IsMap()) { Logger::Log(LogLevel::Error, "Config: While resolving dependencies:\nDescription of " "repository {} is not a map", desc_str); return std::nullopt; } if (not new_repo_desc->At("repository")) { Logger::Log(LogLevel::Error, "Config: While resolving dependencies:\nKey \"repository\" " "missing for repository {}", desc_str); return std::nullopt; } return ResolveRepo(new_repo_desc->At("repository")->get(), repos, seen); } auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos) noexcept -> std::optional { std::unordered_set seen = {}; try { return ResolveRepo(repo_desc, repos, &seen); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Config: While resolving dependencies:\n{}", e.what()); return std::nullopt; } } } // namespace JustMR::Utils just-buildsystem-justbuild-b1fb5fa/src/other_tools/just_mr/utils.hpp000066400000000000000000000147661516554100600263210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/config.hpp" /* Paths and constants required by just-mr */ auto const kDefaultJustPath = "just"; auto const kDefaultGitPath = "git"; auto const kDefaultRCPath = FileSystemManager::GetUserHome() / ".just-mrrc"; auto const kDefaultBuildRoot = StorageConfig::kDefaultBuildRoot; auto const kDefaultCheckoutLocationsFile = FileSystemManager::GetUserHome() / ".just-local.json"; auto const kDefaultDistdirs = FileSystemManager::GetUserHome() / ".distfiles"; std::vector const kTakeOver = {"bindings", "target_file_name", "rule_file_name", "expression_file_name"}; struct JustSubCmdFlags { bool config; // requires setup bool build_root; // supports the local build root arg bool launch; // supports the local launcher arg bool defines; // supports defines arg bool remote; // supports remote exec args, including client-side auth bool remote_props; // supports remote-execution properties bool serve; // supports a serve endpoint bool dispatch; // supports dispatching of the remote-execution endpoint bool does_build; // is actually building something, i.e., supports options // related to the build result }; // ordered, so that we have replicability std::map const kKnownJustSubcommands{ {"version", {.config = false, .build_root = false, .launch = false, .defines = false, .remote = false, .remote_props = false, .serve = false, .dispatch = false, .does_build = false}}, {"describe", {.config = true, .build_root = true, .launch = false, .defines = true, .remote = true, .remote_props = false, .serve = true, .dispatch = false, .does_build = false}}, {"analyse", {.config = true, .build_root = true, .launch = true, .defines = true, .remote = true, .remote_props = true, .serve = true, .dispatch = true, .does_build = false}}, {"build", {.config = true, .build_root = true, .launch = true, .defines = true, .remote = true, .remote_props = true, .serve = true, .dispatch = true, .does_build = true}}, {"install", {.config = true, .build_root = true, .launch = true, .defines = true, .remote = true, .remote_props = true, .serve = true, .dispatch = true, .does_build = true}}, {"rebuild", {.config = true, .build_root = true, .launch = true, .defines = true, .remote = true, .remote_props = true, .serve = true, .dispatch = true, .does_build = true}}, {"add-to-cas", {.config = false, .build_root = true, .launch = false, .defines = false, .remote = true, .remote_props = false, .serve = false, .dispatch = false, .does_build = false}}, {"install-cas", {.config = false, .build_root = true, .launch = false, .defines = false, .remote = true, .remote_props = false, .serve = false, .dispatch = false, .does_build = false}}, {"gc", {.config = false, .build_root = true, .launch = false, .defines = false, .remote = false, .remote_props = false, .serve = false, .dispatch = false, .does_build = false}}}; nlohmann::json const kDefaultConfigLocations = nlohmann::json::array( {{{"root", "workspace"}, {"path", "repos.json"}}, {{"root", "workspace"}, {"path", "etc/repos.json"}}, {{"root", "home"}, {"path", ".just-repos.json"}}, {{"root", "system"}, {"path", "etc/just-repos.json"}}}); /// \brief Checkout type enum enum class CheckoutType : std::uint8_t { Git, Archive, ForeignFile, File, Distdir, GitTree, Precomputed }; /// \brief Checkout type map std::unordered_map const kCheckoutTypeMap = { {"git", CheckoutType::Git}, {"archive", CheckoutType::Archive}, {"zip", CheckoutType::Archive}, // treated the same as "archive" {"foreign file", CheckoutType::ForeignFile}, {"file", CheckoutType::File}, {"distdir", CheckoutType::Distdir}, {"git tree", CheckoutType::GitTree}, {"computed", CheckoutType::Precomputed}, {"tree structure", CheckoutType::Precomputed}}; namespace JustMR::Utils { /// \brief Recursive part of the ResolveRepo function. /// Keeps track of repository names to detect any cyclic dependencies. /// \param repos ExpressionPtr of Map type. [[nodiscard]] auto ResolveRepo( ExpressionPtr const& repo_desc, ExpressionPtr const& repos, gsl::not_null*> const& seen) -> std::optional; /// \brief Resolves any cyclic dependency issues and follows the repository /// dependencies until the one containing the workspace root is found. /// Returns a repository entry as an ExpressionPtr, or nullopt if cyclic /// dependency found. /// \param repos ExpressionPtr of Map type. [[nodiscard]] auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos) noexcept -> std::optional; } // namespace JustMR::Utils #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/000077500000000000000000000000001516554100600245705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/TARGETS000066400000000000000000000134771516554100600256400ustar00rootroot00000000000000{ "critical_git_op_map": { "type": ["@", "rules", "CC", "library"] , "name": ["critical_git_op_map"] , "hdrs": ["critical_git_op_map.hpp"] , "srcs": ["critical_git_op_map.cpp"] , "deps": [ ["src/buildtool/multithreading", "async_map_consumer"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [["src/other_tools/git_operations", "git_operations"]] } , "import_to_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["import_to_git_map"] , "hdrs": ["import_to_git_map.hpp"] , "srcs": ["import_to_git_map.cpp"] , "deps": [ "critical_git_op_map" , ["@", "gsl", "", "gsl"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/multithreading", "task_system"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "tmp_dir"] ] } , "git_update_map": { "type": ["@", "rules", "CC", "library"] , "name": ["git_update_map"] , "hdrs": ["git_update_map.hpp"] , "srcs": ["git_update_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/storage", "config"] , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/other_tools/git_operations", "git_repo_remote"] ] } , "content_cas_map": { "type": ["@", "rules", "CC", "library"] , "name": ["content_cas_map"] , "hdrs": ["content_cas_map.hpp"] , "srcs": ["content_cas_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/utils", "content"] , ["src/utils/cpp", "expected"] ] } , "archive_fetch_map": { "type": ["@", "rules", "CC", "library"] , "name": ["archive_fetch_map"] , "hdrs": ["archive_fetch_map.hpp"] , "srcs": ["archive_fetch_map.cpp"] , "deps": [ "content_cas_map" , ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] ] } , "git_tree_fetch_map": { "type": ["@", "rules", "CC", "library"] , "name": ["git_tree_fetch_map"] , "hdrs": ["git_tree_fetch_map.hpp"] , "srcs": ["git_tree_fetch_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/system", "system_command"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "tmp_dir"] ] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/archive_fetch_map.cpp000066400000000000000000000124031516554100600307230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/archive_fetch_map.hpp" #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" namespace { void ProcessContent(std::filesystem::path const& content_path, std::filesystem::path const& target_name, gsl::not_null const& local_api, IExecutionApi const* remote_api, ArtifactDigest const& content_digest, gsl::not_null const& stats, ArchiveFetchMap::SetterPtr const& setter, ArchiveFetchMap::LoggerPtr const& logger) { // try to back up to remote CAS if (remote_api != nullptr) { if (not local_api->RetrieveToCas( {Artifact::ObjectInfo{.digest = content_digest, .type = ObjectType::File}}, *remote_api)) { // give a warning (*logger)(fmt::format("Failed to back up content {} from local CAS " "to remote", content_digest.hash()), /*fatal=*/false); } } // then, copy content into fetch_dir if (FileSystemManager::Exists(target_name)) { std::filesystem::permissions(target_name, std::filesystem::perms::owner_write, std::filesystem::perm_options::add); } if (not FileSystemManager::CopyFile(content_path, target_name)) { (*logger)(fmt::format("Failed to copy content {} from CAS to {}", content_digest.hash(), target_name.string()), /*fatal=*/true); return; } // success stats->IncrementExecutedCounter(); (*setter)(true); } } // namespace auto CreateArchiveFetchMap(gsl::not_null const& content_cas_map, std::filesystem::path const& fetch_dir, gsl::not_null const& storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, gsl::not_null const& stats, std::size_t jobs) -> ArchiveFetchMap { auto fetch_archive = [content_cas_map, fetch_dir, storage, local_api, remote_api, stats](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { // get corresponding distfile auto distfile = (key.distfile ? key.distfile.value() : std::filesystem::path(key.fetch_url).filename().string()); auto target_name = fetch_dir / distfile; // make sure content is in CAS content_cas_map->ConsumeAfterKeysReady( ts, {key}, [target_name, storage, local_api, remote_api, hash_info = key.content_hash, stats, setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now auto const& cas = storage->CAS(); ArtifactDigest const digest{hash_info, 0}; auto content_path = cas.BlobPath(digest, /*is_executable=*/false) .value(); ProcessContent(content_path, target_name, local_api, remote_api, digest, stats, setter, logger); }, [logger, hash = key.content_hash.Hash()](auto const& msg, bool fatal) { (*logger)( fmt::format( "While ensuring content {} is in CAS:\n{}", hash, msg), fatal); }); }; return AsyncMapConsumer(fetch_archive, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/archive_fetch_map.hpp000066400000000000000000000037651516554100600307430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" /// \brief Maps an archive content hash to a status flag. using ArchiveFetchMap = AsyncMapConsumer; [[nodiscard]] auto CreateArchiveFetchMap( gsl::not_null const& content_cas_map, std::filesystem::path const& fetch_dir, // should exist! gsl::not_null const& storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, gsl::not_null const& stats, std::size_t jobs) -> ArchiveFetchMap; // use explicit cast to std::function to allow template deduction when used static const std::function kArchiveContentPrinter = [](ArchiveContent const& x) -> std::string { return x.content_hash.Hash(); }; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/content_cas_map.cpp000066400000000000000000000412231516554100600304330ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/content_cas_map.hpp" #include #include #include // std::move #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/utils/content.hpp" #include "src/utils/cpp/expected.hpp" namespace { void FetchFromNetwork(ArchiveContent const& key, MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, Storage const& native_storage, gsl::not_null const& progress, ContentCASMap::SetterPtr const& setter, ContentCASMap::LoggerPtr const& logger) { // first, check that mandatory fields are provided if (key.fetch_url.empty()) { (*logger)("Failed to provide archive fetch url!", /*fatal=*/true); return; } // now do the actual fetch auto data = NetworkFetchWithMirrors( key.fetch_url, key.mirrors, ca_info, additional_mirrors); if (not data) { (*logger)(fmt::format("Failed to fetch a file with id {} from provided " "remotes:{}", key.content_hash.Hash(), data.error()), /*fatal=*/true); return; } // check content wrt checksums if (key.sha256) { auto actual_sha256 = GetContentHash(*data); if (actual_sha256 != key.sha256.value()) { (*logger)(fmt::format("SHA256 mismatch for {}: expected {}, got {}", key.fetch_url, key.sha256.value(), actual_sha256), /*fatal=*/true); return; } } if (key.sha512) { auto actual_sha512 = GetContentHash(*data); if (actual_sha512 != key.sha512.value()) { (*logger)(fmt::format("SHA512 mismatch for {}: expected {}, got {}", key.fetch_url, key.sha512.value(), actual_sha512), /*fatal=*/true); return; } } // add the fetched data to native CAS auto path = StorageUtils::AddToCAS(native_storage, *data); // check one last time if content is in native CAS now if (not path) { (*logger)(fmt::format("Failed to store fetched content from {}", key.fetch_url), /*fatal=*/true); return; } // check that the data we stored actually produces the requested digest auto const& native_cas = native_storage.CAS(); if (not native_cas.BlobPath(ArtifactDigest{key.content_hash, 0}, /*is_executable=*/false)) { (*logger)( fmt::format("Content {} was not found at given fetch location {}", key.content_hash.Hash(), key.fetch_url), /*fatal=*/true); return; } progress->TaskTracker().Stop(key.origin); // success! (*setter)(nullptr); } } // namespace auto CreateContentCASMap( LocalPathsPtr const& just_mr_paths, MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& native_storage, Storage const* compat_storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, gsl::not_null const& progress, std::size_t jobs) -> ContentCASMap { auto ensure_in_cas = [just_mr_paths, additional_mirrors, ca_info, critical_git_op_map, serve, native_storage_config, compat_storage_config, native_storage, compat_storage, local_api, remote_api, progress](auto ts, auto setter, auto logger, auto /*unused*/, auto const& key) { auto const native_digest = ArtifactDigest{key.content_hash, 0}; // check native local CAS if (local_api->IsAvailable(native_digest)) { (*setter)(nullptr); return; } // check if content is in Git cache; // ensure Git cache GitOpKey op_key = { .params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, native_digest, just_mr_paths, additional_mirrors, ca_info, serve, native_storage_config, compat_storage_config, native_storage, compat_storage, local_api, remote_api, progress, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } auto const just_git_cas = op_result.git_cas; // open fake repo wrap for GitCAS auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)("Could not open Git cache repository!", /*fatal=*/true); return; } // verify if local Git knows content blob auto wrapped_logger = std::make_shared( [&logger, hash = key.content_hash.Hash()](auto const& msg, bool fatal) { (*logger)(fmt::format("While verifying presence of " "blob {}:\n{}", hash, msg), fatal); }); auto res = just_git_repo->TryReadBlob(key.content_hash.Hash(), wrapped_logger); if (not res.first) { // blob check failed return; } auto const& native_cas = native_storage->CAS(); if (res.second) { // blob found; add it to native CAS if (not native_cas.StoreBlob(*res.second, /*is_executable=*/false)) { (*logger)(fmt::format("Failed to store content {} " "to native local CAS", key.content_hash.Hash()), /*fatal=*/true); return; } // content stored to native CAS (*setter)(nullptr); return; } // check for blob in older generations for (std::size_t generation = 1; generation < native_storage_config->num_generations; generation++) { auto old = native_storage_config->GitGenerationRoot(generation); if (FileSystemManager::IsDirectory(old)) { auto old_repo = GitRepo::Open(old); auto no_logging = std::make_shared( [](auto /*unused*/, auto /*unused*/) {}); if (old_repo) { auto res = old_repo->TryReadBlob( key.content_hash.Hash(), no_logging); if (res.first and res.second) { // read blob from older generation if (not native_cas.StoreBlob( *res.second, /*is_executable=*/false)) { (*logger)(fmt::format( "Failed to store content {} " "to native local CAS", key.content_hash.Hash()), /*fatal=*/true); return; } // content stored in native CAS (*setter)(nullptr); return; } } } } // blob not found in Git cache progress->TaskTracker().Start(key.origin); // add distfile to native CAS auto repo_distfile = (key.distfile ? key.distfile.value() : std::filesystem::path(key.fetch_url) .filename() .string()); StorageUtils::AddDistfileToCAS( *native_storage, repo_distfile, just_mr_paths); // check if content is in native CAS now if (native_cas.BlobPath(native_digest, /*is_executable=*/false)) { progress->TaskTracker().Stop(key.origin); (*setter)(nullptr); return; } // check if content is known to remote serve service if (serve != nullptr and remote_api != nullptr) { auto const remote_digest = serve->ContentInRemoteCAS(key.content_hash.Hash()); // try to get content from remote CAS if (remote_digest and remote_api->RetrieveToCas( {Artifact::ObjectInfo{.digest = *remote_digest, .type = ObjectType::File}}, *local_api)) { progress->TaskTracker().Stop(key.origin); if (remote_digest->hash() == key.content_hash.Hash()) { // content is in native local CAS, so all done (*setter)(nullptr); return; } // if content is in compatible local CAS, rehash it if (compat_storage_config == nullptr or compat_storage == nullptr) { // sanity check (*logger)("No compatible local storage set up!", /*fatal=*/true); return; } auto const& compat_cas = compat_storage->CAS(); auto const cas_path = compat_cas.BlobPath( *remote_digest, /*is_executable=*/false); if (not cas_path) { (*logger)(fmt::format("Expected content {} not " "found in " "compatible local CAS", remote_digest->hash()), /*fatal=*/true); return; } auto rehashed_digest = native_cas.StoreBlob( *cas_path, /*is_executable=*/false); if (not rehashed_digest or rehashed_digest->hash() != key.content_hash.Hash()) { (*logger)(fmt::format("Failed to rehash content {} " "into native local CAS", remote_digest->hash()), /*fatal=*/true); return; } // cache association between digests auto error_msg = RehashUtils::StoreRehashedDigest( native_digest, *rehashed_digest, ObjectType::File, *native_storage_config, *compat_storage_config); if (error_msg) { (*logger)(fmt::format("Failed to cache digests " "mapping with:\n{}", *error_msg), /*fatal=*/true); return; } // content is in native local CAS now (*setter)(nullptr); return; } } // check if content is on remote, if given and native if (compat_storage_config == nullptr and remote_api != nullptr and remote_api->RetrieveToCas( {Artifact::ObjectInfo{.digest = native_digest, .type = ObjectType::File}}, *local_api)) { progress->TaskTracker().Stop(key.origin); (*setter)(nullptr); return; } // revert to network fetch FetchFromNetwork(key, additional_mirrors, ca_info, *native_storage, progress, setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); }; return AsyncMapConsumer(ensure_in_cas, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/content_cas_map.hpp000066400000000000000000000121671516554100600304450ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CONTENT_CAS_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CONTENT_CAS_MAP_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/utils/cpp/hash_combine.hpp" struct ArchiveContent { HashInfo content_hash; /* key */ std::optional distfile{std::nullopt}; std::string fetch_url; std::vector mirrors; std::optional sha256{std::nullopt}; std::optional sha512{std::nullopt}; // name of repository for which work is done; used in progress reporting std::string origin; [[nodiscard]] auto operator==(const ArchiveContent& other) const -> bool { return content_hash.Hash() == other.content_hash.Hash(); } }; // Used in callers of ContentCASMap which need extra fields struct ArchiveRepoInfo { ArchiveContent archive; /* key */ std::string repo_type; /* key */ std::string subdir; /* key */ // create root based on "special" pragma value std::optional pragma_special{std::nullopt}; /* key */ // create an absent root bool absent{}; /* key */ [[nodiscard]] auto operator==(const ArchiveRepoInfo& other) const -> bool { return archive == other.archive and repo_type == other.repo_type and subdir == other.subdir and pragma_special == other.pragma_special and absent == other.absent; } }; struct ForeignFileInfo { ArchiveContent archive; /* key */ std::string name; /* key */ bool executable{}; /* key */ bool absent{}; /* key */ [[nodiscard]] auto operator==(const ForeignFileInfo& other) const -> bool { return archive == other.archive and name == other.name and executable == other.executable and absent == other.absent; } }; /// \brief Maps the content hash of an archive to nullptr, as we only care if /// the map fails or not. using ContentCASMap = AsyncMapConsumer; [[nodiscard]] auto CreateContentCASMap( LocalPathsPtr const& just_mr_paths, MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& native_storage, Storage const* compat_storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, gsl::not_null const& progress, std::size_t jobs) -> ContentCASMap; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const ArchiveContent& ct) const noexcept -> std::size_t { return std::hash{}(ct.content_hash.Hash()); } }; // Used in callers of ContentCASMap which need extra fields template <> struct hash { [[nodiscard]] auto operator()(const ArchiveRepoInfo& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.archive); hash_combine(&seed, ct.repo_type); hash_combine(&seed, ct.subdir); hash_combine>(&seed, ct.pragma_special); hash_combine(&seed, ct.absent); return seed; } }; template <> struct hash { [[nodiscard]] auto operator()(const ForeignFileInfo& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.archive); hash_combine(&seed, ct.name); hash_combine(&seed, ct.executable); hash_combine(&seed, ct.absent); return seed; } }; } // namespace std #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CONTENT_CAS_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/critical_git_op_map.cpp000066400000000000000000000042451516554100600312710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include #include "src/other_tools/git_operations/git_operations.hpp" // define the mapping to actual operations being called GitOpKeyMap const GitOpKey::kMap = { {GitOpType::INITIAL_COMMIT, CriticalGitOps::GitInitialCommit}, {GitOpType::ENSURE_INIT, CriticalGitOps::GitEnsureInit}, {GitOpType::KEEP_TAG, CriticalGitOps::GitKeepTag}, {GitOpType::GET_HEAD_ID, CriticalGitOps::GitGetHeadId}, {GitOpType::KEEP_TREE, CriticalGitOps::GitKeepTree}}; /// \brief Create a CriticalOpMap object auto CreateCriticalGitOpMap(CriticalGitOpGuardPtr const& crit_git_op_ptr) -> CriticalGitOpMap { auto crit_op_runner = [crit_git_op_ptr](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { auto curr_key = crit_git_op_ptr->FetchAndSetCriticalKey(key); if (curr_key == std::nullopt) { // do critical operation now (*setter)(key.operation(key.params, logger)); } else { // do critical operation after curr_key was processed (*subcaller)( {*curr_key}, [key, setter, logger](auto const& /*unused*/) { (*setter)(key.operation(key.params, logger)); }, logger); } }; return AsyncMapConsumer(crit_op_runner); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/critical_git_op_map.hpp000066400000000000000000000103011516554100600312640ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CRITICAL_GIT_OP_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CRITICAL_GIT_OP_MAP_HPP #include #include #include #include #include #include #include #include #include #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" using GitOpKeyMap = std::unordered_map< GitOpType, std::function>; struct GitOpKey { GitOpParams params{"", ""}; /* key (with exceptions) */ GitOpType op_type{GitOpType::DEFAULT_OP}; /* key */ [[nodiscard]] auto operation(GitOpParams const& params, AsyncMapConsumerLoggerPtr const& logger) const -> GitOpValue { return kMap.at(op_type)(params, logger); } [[nodiscard]] auto operator==(GitOpKey const& other) const -> bool { return params == other.params && op_type == other.op_type; } private: static GitOpKeyMap const kMap; }; class CriticalGitOpGuard; using CriticalGitOpGuardPtr = std::shared_ptr; using CriticalGitOpMap = AsyncMapConsumer; [[nodiscard]] auto CreateCriticalGitOpMap( CriticalGitOpGuardPtr const& crit_git_op_ptr) -> CriticalGitOpMap; /// \brief Class ensuring thread safety in critical Git operations. /// By always storing the most recent operation to be executed, a chain of /// operations can be created such that no thread is left in a blocking state. /// Each repo has its own key, so the caller has to ensure the target_path /// parameter provided is non-empty. class CriticalGitOpGuard { public: [[nodiscard]] auto FetchAndSetCriticalKey(GitOpKey const& new_key) -> std::optional { // making sure only one thread at a time processes this section std::scoped_lock const lock(critical_key_mutex_); // try emplace a new value auto const canonical_path = std::filesystem::weakly_canonical(new_key.params.target_path); auto result = curr_critical_key_.try_emplace(canonical_path, new_key); // If the insertion happens, there are no keys to wait for. std::nullopt // is returned. if (result.second) { return std::nullopt; } // If insertion fails (the path is being processed), mark the path // occupied by the new key, but return the previous key to inform // the new task to wait for the previous operation to complete. return std::exchange(result.first->second, new_key); } private: std::unordered_map curr_critical_key_; std::mutex critical_key_mutex_; }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(GitOpParams const& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.target_path); hash_combine(&seed, ct.git_hash); return seed; } }; template <> struct hash { [[nodiscard]] auto operator()(GitOpKey const& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.params); hash_combine<>(&seed, ct.op_type); return seed; } }; } // namespace std #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_CRITICAL_GIT_OP_MAP_HPPjust-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/git_tree_fetch_map.cpp000066400000000000000000001060031516554100600311040ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include #include #include #include #include #include // std::move #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { void BackupToRemote(ArtifactDigest const& digest, StorageConfig const* native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& local_api, IExecutionApi const& remote_api, GitTreeFetchMap::LoggerPtr const& logger) { // try to back up to remote CAS auto repo = RepositoryConfig{}; if (repo.SetGitCAS(native_storage_config->GitRoot(), native_storage_config)) { auto git_api = MRGitApi{&repo, native_storage_config, compat_storage_config, compat_storage_config != nullptr ? &*local_api : nullptr}; if (git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, remote_api)) { return; } } // give a warning (*logger)(fmt::format("Failed to back up tree {} from local CAS to remote", digest.hash()), /*fatal=*/false); } /// \brief Moves the root tree from local CAS to the Git cache and sets the /// root. void MoveCASTreeToGit( HashInfo const& tree_hash, ArtifactDigest const& digest, // native or compatible gsl::not_null const& import_to_git_map, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null const& ts, GitTreeFetchMap::SetterPtr const& setter, GitTreeFetchMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage auto tmp_dir = native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", tree_hash.Hash()), true); return; } if (not local_api->RetrieveToPaths( {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { (*logger)(fmt::format("Failed to copy git-tree {} to {}", tree_hash.Hash(), tmp_dir->GetPath().string()), true); return; } CommitInfo c_info{tmp_dir->GetPath(), "tree", tree_hash.Hash()}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive tree_hash, native_storage_config, compat_storage_config, local_api, remote_api, backup_to_remote, setter, logger](auto const& values) { if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { // back up only native digests, as that is what Git stores auto const native_digest = ArtifactDigest{tree_hash, 0}; BackupToRemote(native_digest, native_storage_config, compat_storage_config, local_api, *remote_api, logger); } (*setter)(false /*no cache hit*/); }, [logger, tmp_dir, tree_hash](auto const& msg, bool fatal) { (*logger)(fmt::format( "While moving git-tree {} from {} to local git:\n{}", tree_hash.Hash(), tmp_dir->GetPath().string(), msg), fatal); }); } void TagAndSetRoot( ArtifactDigest const& digest, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& critical_git_op_map, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null const& ts, GitTreeFetchMap::SetterPtr const& setter, GitTreeFetchMap::LoggerPtr const& logger) { auto repo = native_storage_config->GitRoot(); GitOpKey op_key = {.params = { repo, // target_path digest.hash(), // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [digest, backup_to_remote, native_storage_config, compat_storage_config, local_api, remote_api, logger, setter](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Tree tagging failed", /*fatal=*/true); return; } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { BackupToRemote(digest, native_storage_config, compat_storage_config, local_api, *remote_api, logger); } (*setter)(false /*no cache hit*/); }, [logger, repo, digest](auto const& msg, bool fatal) { (*logger)(fmt::format("While tagging tree {} in {} to keep it " "alive:\n{}", digest.hash(), repo.string(), msg), fatal); }); } void TakeTreeFromOlderGeneration( std::size_t generation, ArtifactDigest const& digest, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, GitCASPtr const& git_cas, gsl::not_null const& critical_git_op_map, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null const& ts, GitTreeFetchMap::SetterPtr const& setter, GitTreeFetchMap::LoggerPtr const& logger) { auto source = native_storage_config->GitGenerationRoot(generation); GitOpKey op_key = {.params = { source, // target_path digest.hash(), // git_hash "Tag commit for fetching" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [digest, git_cas, critical_git_op_map, local_api, remote_api, backup_to_remote, ts, setter, logger, source, native_storage_config, compat_storage_config](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Tree tagging failed", /*fatal=*/true); return; } auto tag = *op_result.result; auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { (*logger)("Could not open main git repository", /*fatal=*/true); return; } auto fetch_logger = std::make_shared( [logger, tag, source](auto const& msg, bool fatal) { (*logger)(fmt::format("While fetching {} from {}:\n{}", tag, source.string(), msg), fatal); }); if (not git_repo->LocalFetchViaTmpRepo( *native_storage_config, source, tag, fetch_logger)) { return; } TagAndSetRoot(digest, native_storage_config, compat_storage_config, critical_git_op_map, local_api, remote_api, backup_to_remote, ts, setter, logger); }, [logger, source, digest](auto const& msg, bool fatal) { (*logger)( fmt::format("While tagging tree {} in {} for fetching:\n{}", source.string(), digest.hash(), msg), fatal); }); } } // namespace auto CreateGitTreeFetchMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, std::string const& git_bin, std::vector const& launcher, MirrorsPtr const& mirrors, ServeApi const* serve, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null const& progress, std::size_t jobs) -> GitTreeFetchMap { auto tree_to_cache = [critical_git_op_map, import_to_git_map, git_bin, launcher, mirrors, serve, native_storage_config, compat_storage_config, local_api, remote_api, backup_to_remote, progress](auto ts, auto setter, auto logger, auto /*unused*/, auto const& key) { // check whether tree exists already in Git cache; // ensure Git cache exists GitOpKey op_key = { .params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [critical_git_op_map, import_to_git_map, git_bin, launcher, mirrors, serve, native_storage_config, compat_storage_config, local_api, remote_api, backup_to_remote, key, progress, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git cache init failed", /*fatal=*/true); return; } // Open fake tmp repo to check if tree is known to Git cache auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", native_storage_config->GitRoot().string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While checking tree exists in " "Git cache:\n{}", msg), fatal); }); // check if the desired tree ID is in Git cache auto tree_found = git_repo->CheckTreeExists( key.tree_hash.Hash(), wrapped_logger); if (not tree_found) { // errors encountered return; } if (*tree_found) { // backup to remote if needed if (backup_to_remote and remote_api != nullptr) { BackupToRemote(ArtifactDigest{key.tree_hash, 0}, native_storage_config, compat_storage_config, local_api, *remote_api, logger); } // success (*setter)(true /*cache hit*/); return; } // Check older generations for presence of the tree for (std::size_t generation = 1; generation < native_storage_config->num_generations; generation++) { auto old = native_storage_config->GitGenerationRoot(generation); if (FileSystemManager::IsDirectory(old)) { auto old_repo = GitRepo::Open(old); auto no_logging = std::make_shared( [](auto /*unused*/, auto /*unused*/) {}); if (old_repo) { auto check_result = old_repo->CheckTreeExists( key.tree_hash.Hash(), no_logging); if (check_result and *check_result) { TakeTreeFromOlderGeneration( generation, ArtifactDigest{key.tree_hash, 0}, native_storage_config, compat_storage_config, op_result.git_cas, critical_git_op_map, local_api, remote_api, backup_to_remote, ts, setter, logger); return; } } } } // check if tree is known to native local CAS auto const native_digest = ArtifactDigest{key.tree_hash, 0}; if (local_api->IsAvailable(native_digest)) { // import tree to Git cache MoveCASTreeToGit(key.tree_hash, native_digest, import_to_git_map, native_storage_config, compat_storage_config, local_api, remote_api, backup_to_remote, ts, setter, logger); // done! return; } progress->TaskTracker().Start(key.origin); // check if tree is known to remote serve service and can be // provided via the remote CAS if (serve != nullptr and remote_api != nullptr) { auto const remote_digest = serve->TreeInRemoteCAS(key.tree_hash.Hash()); // try to get content from remote CAS into local CAS; // whether it is retrieved locally in native or // compatible CAS, it will be imported to Git either way if (remote_digest and remote_api->RetrieveToCas( {Artifact::ObjectInfo{.digest = *remote_digest, .type = ObjectType::Tree}}, *local_api)) { progress->TaskTracker().Stop(key.origin); MoveCASTreeToGit(key.tree_hash, *remote_digest, import_to_git_map, native_storage_config, compat_storage_config, local_api, remote_api, false, // tree already on remote, // so ignore backing up ts, setter, logger); // done! return; } } // check if tree is on remote, if given and native if (compat_storage_config == nullptr and remote_api != nullptr and remote_api->RetrieveToCas( {Artifact::ObjectInfo{.digest = native_digest, .type = ObjectType::Tree}}, *local_api)) { progress->TaskTracker().Stop(key.origin); MoveCASTreeToGit(key.tree_hash, native_digest, import_to_git_map, native_storage_config, compat_storage_config, local_api, remote_api, false, // tree already on remote, // so ignore backing up ts, setter, logger); // done! return; } // create temporary location for command execution root auto content_dir = native_storage_config->CreateTypedTmpDir("git-tree"); if (not content_dir) { (*logger)( "Failed to create execution root tmp directory for " "tree id map!", /*fatal=*/true); return; } // create temporary location for storing command result files auto out_dir = native_storage_config->CreateTypedTmpDir("git-tree"); if (not out_dir) { (*logger)( "Failed to create results tmp directory for tree id " "map!", /*fatal=*/true); return; } // execute command in temporary location SystemCommand system{key.tree_hash.Hash()}; auto cmdline = launcher; std::copy(key.command.begin(), key.command.end(), std::back_inserter(cmdline)); auto inherit_env = MirrorsUtils::GetInheritEnv(mirrors, key.inherit_env); std::map env{key.env_vars}; for (auto const& k : inherit_env) { const char* v = std::getenv(k.c_str()); if (v != nullptr) { env[k] = std::string(v); } } auto const exit_code = system.Execute( cmdline, env, content_dir->GetPath(), out_dir->GetPath()); if (not exit_code) { (*logger)(fmt::format("Failed to execute command:\n{}", nlohmann::json(cmdline).dump()), /*fatal=*/true); return; } // create temporary location for the import repository auto repo_dir = native_storage_config->CreateTypedTmpDir("import-repo"); if (not repo_dir) { (*logger)( "Failed to create tmp directory for import repository", /*fatal=*/true); return; } // do an import to git with tree check GitOpKey op_key = { .params = { repo_dir->GetPath(), // target_path "", // git_hash fmt::format("Content of tree {}", key.tree_hash.Hash()), // message content_dir->GetPath() // source_path }, .op_type = GitOpType::INITIAL_COMMIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [repo_dir, // keep repo_dir alive content_dir, // keep content_dir alive out_dir, // keep stdout/stderr of command alive critical_git_op_map, just_git_cas = op_result.git_cas, cmdline, key, git_bin, launcher, mirrors, native_storage_config, compat_storage_config, local_api, remote_api, backup_to_remote, progress, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Commit failed", /*fatal=*/true); return; } // Open fake tmp repository to check for tree auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", repo_dir->GetPath().string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While checking tree " "exists:\n{}", msg), fatal); }); // check that the desired tree ID is part of the repo auto tree_check = git_repo->CheckTreeExists( key.tree_hash.Hash(), wrapped_logger); if (not tree_check) { // errors encountered return; } if (not *tree_check) { std::string out_str{}; std::string err_str{}; auto cmd_out = FileSystemManager::ReadFile( out_dir->GetPath() / "stdout"); auto cmd_err = FileSystemManager::ReadFile( out_dir->GetPath() / "stderr"); if (cmd_out) { out_str = *cmd_out; } if (cmd_err) { err_str = *cmd_err; } std::string output{}; if (not out_str.empty() or not err_str.empty()) { output = fmt::format(".\nOutput of command:\n{}{}", out_str, err_str); } (*logger)( fmt::format("Executing {} did not create " "specified tree {}{}", nlohmann::json(cmdline).dump(), key.tree_hash.Hash(), output), /*fatal=*/true); return; } auto target_path = repo_dir->GetPath(); // fetch all into Git cache auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)( fmt::format( "Could not open Git repository {}", native_storage_config->GitRoot().string()), /*fatal=*/true); return; } // define temp repo path auto tmp_dir = native_storage_config->CreateTypedTmpDir( "git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Could not create unique " "path for target {}", target_path.string()), /*fatal=*/true); return; } wrapped_logger = std::make_shared( [logger, target_path](auto const& msg, bool fatal) { (*logger)( fmt::format("While fetch via tmp repo " "for target {}:\n{}", target_path.string(), msg), fatal); }); if (not just_git_repo->FetchViaTmpRepo( *native_storage_config, target_path.string(), std::nullopt, key.inherit_env, git_bin, launcher, wrapped_logger)) { return; } // setup a wrapped_logger wrapped_logger = std::make_shared( [logger, target_path](auto const& msg, bool fatal) { (*logger)( fmt::format("While doing keep commit " "and setting Git tree for " "target {}:\n{}", target_path.string(), msg), fatal); }); // Keep tag for commit GitOpKey op_key = { .params = { native_storage_config ->GitRoot(), // target_path *op_result.result, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TAG}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [remote_api, native_storage_config, compat_storage_config, local_api, backup_to_remote, key, progress, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } progress->TaskTracker().Stop(key.origin); // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { BackupToRemote( ArtifactDigest{key.tree_hash, 0}, native_storage_config, compat_storage_config, local_api, *remote_api, logger); } // success (*setter)(false /*no cache hit*/); }, [logger, commit = *op_result.result, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "KEEP_TAG for commit {} in " "repository {}:\n{}", commit, target_path.string(), msg), fatal); }); }, [logger, target_path = repo_dir->GetPath()](auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "INITIAL_COMMIT for target {}:\n{}", target_path.string(), msg), fatal); }); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT bare for target {}:\n{}", target_path.string(), msg), fatal); }); }; return AsyncMapConsumer(tree_to_cache, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/git_tree_fetch_map.hpp000066400000000000000000000063101516554100600311110ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" // Stores all the information needed to make a Git tree available struct GitTreeInfo { HashInfo tree_hash; /* key */ std::map env_vars; std::vector inherit_env; std::vector command; // name of repository for which work is done; used in progress reporting std::string origin; [[nodiscard]] auto operator==(const GitTreeInfo& other) const -> bool { return tree_hash.Hash() == other.tree_hash.Hash(); } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const GitTreeInfo& gti) const noexcept -> std::size_t { return std::hash{}(gti.tree_hash.Hash()); } }; } // namespace std /// \brief Maps a known tree provided through a generic command to a flag /// signaling if there was a cache hit (i.e., tree was already present). using GitTreeFetchMap = AsyncMapConsumer; [[nodiscard]] auto CreateGitTreeFetchMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, std::string const& git_bin, std::vector const& launcher, MirrorsPtr const& mirrors, ServeApi const* serve, gsl::not_null const& native_storage_config, StorageConfig const* compat_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null const& progress, std::size_t jobs) -> GitTreeFetchMap; // use explicit cast to std::function to allow template deduction when used static const std::function kGitTreeInfoPrinter = [](GitTreeInfo const& x) -> std::string { return x.tree_hash.Hash(); }; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/git_update_map.cpp000066400000000000000000000064161516554100600302650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/git_update_map.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" auto CreateGitUpdateMap( GitCASPtr const& git_cas, std::string const& git_bin, std::vector const& launcher, MirrorsPtr const& mirrors, gsl::not_null const& storage_config, gsl::not_null const& stats, gsl::not_null const& progress, std::size_t jobs) -> GitUpdateMap { auto update_commits = [git_cas, git_bin, launcher, mirrors, storage_config, stats, progress]( auto /* unused */, auto setter, auto logger, auto /* unused */, auto const& key) { // perform git update commit auto git_repo = GitRepoRemote::Open(git_cas); // wrap the tmp odb if (not git_repo) { (*logger)(fmt::format( "Failed to open tmp Git repository for remote {}", key.repo), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While updating commit from remote:\n{}", msg), fatal); }); auto inherit_env = MirrorsUtils::GetInheritEnv(mirrors, key.inherit_env); // update commit auto id = fmt::format("{}:{}", key.repo, key.branch); progress->TaskTracker().Start(id); auto new_commit = git_repo->UpdateCommitViaTmpRepo(*storage_config, key.repo, key.branch, inherit_env, git_bin, launcher, wrapped_logger); progress->TaskTracker().Stop(id); if (not new_commit) { return; } stats->IncrementExecutedCounter(); (*setter)(new_commit->c_str()); }; return AsyncMapConsumer( update_commits, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/git_update_map.hpp000066400000000000000000000053641516554100600302730ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_UPDATE_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_UPDATE_MAP_HPP #include #include #include #include #include "gsl/gsl" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/utils/cpp/hash_combine.hpp" struct RepoDescriptionForUpdating { std::string repo; std::string branch; std::vector inherit_env; /*non-key!*/ [[nodiscard]] auto operator==(const RepoDescriptionForUpdating& other) const -> bool { return repo == other.repo and branch == other.branch; } }; /// \brief Maps a pair of repository url and branch to an updated commit hash. using GitUpdateMap = AsyncMapConsumer; namespace std { template <> struct hash { [[nodiscard]] auto operator()( RepoDescriptionForUpdating const& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.repo); hash_combine(&seed, ct.branch); return seed; } }; } // namespace std [[nodiscard]] auto CreateGitUpdateMap( GitCASPtr const& git_cas, std::string const& git_bin, std::vector const& launcher, MirrorsPtr const& mirrors, gsl::not_null const& storage_config, // native storage config gsl::not_null const& stats, gsl::not_null const& progress, std::size_t jobs) -> GitUpdateMap; // use explicit cast to std::function to allow template deduction when used static const std::function kRepoDescriptionPrinter = [](RepoDescriptionForUpdating const& x) -> std::string { return x.repo; }; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_UPDATE_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/import_to_git_map.cpp000066400000000000000000000265551516554100600310250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include #include #include "fmt/core.h" #include "src/buildtool/multithreading/task_system.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { void KeepCommitAndSetTree( gsl::not_null const& critical_git_op_map, std::string const& commit, GitCASPtr const& just_git_cas, StorageConfig const& storage_config, gsl::not_null const& ts, ImportToGitMap::SetterPtr const& setter, ImportToGitMap::LoggerPtr const& logger) { // Keep tag for commit GitOpKey op_key = {.params = { storage_config.GitRoot(), // target_path commit, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TAG}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [commit, just_git_cas, storage_config, setter, logger]( auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)(fmt::format("Could not open Git repository {}", storage_config.GitRoot().string()), /*fatal=*/true); return; } // get tree id and return it auto wrapped_logger = std::make_shared( [logger, commit](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subtree from commit {}:\n{}", commit, msg), fatal); }); auto res = just_git_repo->GetSubtreeFromCommit( commit, ".", wrapped_logger); if (not res) { return; } (*setter)(std::pair(*std::move(res), just_git_cas)); }, [logger, commit, target_path = storage_config.GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op KEEP_TAG for " "commit {} in target {}:\n{}", commit, target_path.string(), msg), fatal); }); } } // namespace auto CreateImportToGitMap( gsl::not_null const& critical_git_op_map, std::string const& git_bin, std::vector const& launcher, gsl::not_null const& storage_config, std::size_t jobs) -> ImportToGitMap { auto import_to_git = [critical_git_op_map, git_bin, launcher, storage_config](auto ts, auto setter, auto logger, auto /*unused*/, auto const& key) { // The repository path that imports the content must be separate from // the content path, to avoid polluting the entries auto repo_dir = storage_config->CreateTypedTmpDir("import-repo"); if (not repo_dir) { (*logger)(fmt::format("Failed to create import repository tmp " "directory for target {}", key.target_path.string()), true); return; } // Commit content from target_path via the tmp repository GitOpKey op_key = {.params = { repo_dir->GetPath(), // target_path "", // git_hash fmt::format("Content of {} {}", key.repo_type, key.content), // message key.target_path // source_path }, .op_type = GitOpType::INITIAL_COMMIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [repo_dir, // keep repo_dir alive critical_git_op_map, git_bin, launcher, storage_config, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Initial commit failed", /*fatal=*/true); return; } // ensure Git cache // define Git operation to be done GitOpKey op_key = { .params = { storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [repo_dir, // keep repo_dir alive critical_git_op_map, commit = *op_result.result, git_bin, launcher, storage_config, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } // fetch all into Git cache auto just_git_repo = GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { (*logger)( fmt::format("Could not open Git cache " "repository {}", storage_config->GitRoot().string()), /*fatal=*/true); return; } auto const& target_path = repo_dir->GetPath(); auto wrapped_logger = std::make_shared( [logger, target_path](auto const& msg, bool fatal) { (*logger)(fmt::format("While fetch via tmp " "repo from {}:\n{}", target_path.string(), msg), fatal); }); if (not just_git_repo->FetchViaTmpRepo( *storage_config, target_path.string(), std::nullopt, std::vector{} /* inherit_env */, git_bin, launcher, wrapped_logger)) { return; } // setup a wrapped_logger wrapped_logger = std::make_shared( [logger, target_path](auto const& msg, bool fatal) { (*logger)( fmt::format("While doing keep commit " "and setting Git tree for " "target {}:\n{}", target_path.string(), msg), fatal); }); KeepCommitAndSetTree(critical_git_op_map, commit, op_result.git_cas, /*just_git_cas*/ *storage_config, ts, setter, wrapped_logger); }, [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT bare for " "target {}:\n{}", target_path.string(), msg), fatal); }); }, [logger, target_path = key.target_path](auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "INITIAL_COMMIT for target {}:\n{}", target_path.string(), msg), fatal); }); }; return AsyncMapConsumer>( import_to_git, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/ops_maps/import_to_git_map.hpp000066400000000000000000000053301516554100600310160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/path_hash.hpp" struct CommitInfo { std::filesystem::path target_path; /*key*/ std::string repo_type; std::string content; // hash or path CommitInfo(std::filesystem::path const& target_path_, std::string repo_type_, std::string content_) : target_path{std::filesystem::absolute(ToNormalPath(target_path_))}, repo_type{std::move(repo_type_)}, content{std::move(content_)} {}; [[nodiscard]] auto operator==(CommitInfo const& other) const noexcept -> bool { return target_path == other.target_path; } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(CommitInfo const& ct) const noexcept -> std::size_t { return std::hash{}(ct.target_path); } }; } // namespace std /// \brief Maps a directory on the file system to a pair of the tree hash of the /// content of the directory and the Git cache ODB (where the content will be). /// The second entry is set for convenience for follow-up operations (to avoid /// overheads of opening the Git cache), and is nullptr iff the map fails. using ImportToGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateImportToGitMap( gsl::not_null const& critical_git_op_map, std::string const& git_bin, std::vector const& launcher, gsl::not_null const& storage_config, std::size_t jobs) -> ImportToGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPPjust-buildsystem-justbuild-b1fb5fa/src/other_tools/repo_map/000077500000000000000000000000001516554100600245515ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/repo_map/TARGETS000066400000000000000000000031761516554100600256140ustar00rootroot00000000000000{ "repos_to_setup_map": { "type": ["@", "rules", "CC", "library"] , "name": ["repos_to_setup_map"] , "hdrs": ["repos_to_setup_map.hpp"] , "srcs": ["repos_to_setup_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/root_maps", "commit_git_map"] , ["src/other_tools/root_maps", "content_git_map"] , ["src/other_tools/root_maps", "distdir_git_map"] , ["src/other_tools/root_maps", "foreign_file_git_map"] , ["src/other_tools/root_maps", "fpath_git_map"] , ["src/other_tools/root_maps", "tree_id_git_map"] ] , "stage": ["src", "other_tools", "repo_map"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "precomputed_root"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/multithreading", "task_system"] , ["src/other_tools/just_mr", "utils"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/utils", "parse_archive"] , ["src/other_tools/utils", "parse_git_tree"] , ["src/other_tools/utils", "parse_precomputed_root"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "path"] ] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/repo_map/repos_to_setup_map.cpp000066400000000000000000001151041516554100600311660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/repo_map/repos_to_setup_map.hpp" #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/utils/parse_archive.hpp" #include "src/other_tools/utils/parse_git_tree.hpp" #include "src/other_tools/utils/parse_precomputed_root.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/path.hpp" namespace { /// \brief Updates output config with specific keys from input config. void SetReposTakeOver(gsl::not_null const& cfg, ExpressionPtr const& repos, std::string const& repo_name) { if (repos.IsNotNull()) { for (auto const& key : kTakeOver) { auto repos_repo_name = repos->Get(repo_name, Expression::none_t{}); if (repos_repo_name.IsNotNull()) { auto value = repos_repo_name->Get(key, Expression::none_t{}); if (value.IsNotNull()) { (*cfg)[key] = value.ToJson(); } } } } } /// \brief Perform checkout for a Git type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void GitCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& commit_git_map, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { // enforce mandatory fields auto repo_desc_commit = repo_desc->At("commit"); if (not repo_desc_commit) { (*logger)("GitCheckout: Mandatory field \"commit\" is missing", /*fatal=*/true); return; } if (not repo_desc_commit->get()->IsString()) { (*logger)(fmt::format("GitCheckout: Unsupported value {} for mandatory " "field \"commit\"", repo_desc_commit->get()->ToString()), /*fatal=*/true); return; } auto repo_desc_repository = repo_desc->At("repository"); if (not repo_desc_repository) { (*logger)("GitCheckout: Mandatory field \"repository\" is missing", /*fatal=*/true); return; } if (not repo_desc_repository->get()->IsString()) { (*logger)(fmt::format("GitCheckout: Unsupported value {} for mandatory " "field \"repository\"", repo_desc_repository->get()->ToString()), /*fatal=*/true); return; } auto repo_desc_branch = repo_desc->At("branch"); if (not repo_desc_branch) { (*logger)("GitCheckout: Mandatory field \"branch\" is missing", /*fatal=*/true); return; } if (not repo_desc_branch->get()->IsString()) { (*logger)(fmt::format("GitCheckout: Unsupported value {} for mandatory " "field \"branch\"", repo_desc_branch->get()->ToString()), /*fatal=*/true); return; } auto repo_desc_subdir = repo_desc->Get("subdir", Expression::none_t{}); auto subdir = std::filesystem::path((repo_desc_subdir->IsString()) ? repo_desc_subdir->String() : "") .lexically_normal(); if (not PathIsNonUpwards(subdir)) { (*logger)(fmt::format("GitCheckout: Expected field \"subdir\" to be a " "non-upwards path, but found {}", subdir.string()), /*fatal=*/true); return; } // check optional mirrors auto repo_desc_mirrors = repo_desc->Get("mirrors", Expression::list_t{}); std::vector mirrors{}; if (repo_desc_mirrors->IsList()) { mirrors.reserve(repo_desc_mirrors->List().size()); for (auto const& elem : repo_desc_mirrors->List()) { if (not elem->IsString()) { (*logger)(fmt::format("GitCheckout: Unsupported list entry {} " "in optional field \"mirrors\"", elem->ToString()), /*fatal=*/true); return; } mirrors.emplace_back(elem->String()); } } else { (*logger)(fmt::format("GitCheckout: Optional field \"mirrors\" should " "be a list of strings, but found: {}", repo_desc_mirrors->ToString()), /*fatal=*/true); return; } // check "special" pragma auto repo_desc_pragma = repo_desc->At("pragma"); bool const& pragma_is_map = repo_desc_pragma and repo_desc_pragma->get()->IsMap(); auto pragma_special = pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = pragma_special and pragma_special->get()->IsString() and kPragmaSpecialMap.contains(pragma_special->get()->String()) ? std::make_optional( kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // check "absent" pragma auto pragma_absent = pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); std::vector inherit_env{}; auto repo_desc_inherit_env = repo_desc->Get("inherit env", Expression::kEmptyList); if (not repo_desc_inherit_env->IsList()) { (*logger)(fmt::format("GitCheckout: optional field \"inherit env\" " "should be a list of strings, but found {}", repo_desc_inherit_env->ToString()), true); return; } for (auto const& var : repo_desc_inherit_env->List()) { if (not var->IsString()) { (*logger)( fmt::format("GitCheckout: optional field \"inherit env\" " "should be a list of strings, but found entry {}", var->ToString()), true); return; } inherit_env.emplace_back(var->String()); } // populate struct GitRepoInfo git_repo_info = { .hash = repo_desc_commit->get()->String(), .repo_url = repo_desc_repository->get()->String(), .branch = repo_desc_branch->get()->String(), .subdir = subdir.empty() ? "." : subdir.string(), .inherit_env = inherit_env, .mirrors = std::move(mirrors), .origin = repo_name, .ignore_special = pragma_special_value == PragmaSpecial::Ignore, .absent = pragma_absent_value}; // get the WS root as git tree commit_git_map->ConsumeAfterKeysReady( ts, {std::move(git_repo_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = values[0]->first; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); if (values[0]->second) { stats->IncrementCacheHitsCounter(); } else { stats->IncrementExecutedCounter(); } (*setter)(std::move(cfg)); }, [logger, repo_name](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "repository {} of type \"git\":\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); } /// \brief Perform checkout for an archive type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void ArchiveCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, std::string const& repo_type, gsl::not_null const& content_git_map, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { auto archive_repo_info = ParseArchiveDescription(repo_desc, repo_type, repo_name, logger); if (not archive_repo_info) { return; } // get the WS root as git tree content_git_map->ConsumeAfterKeysReady( ts, {std::move(*archive_repo_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = values[0]->first; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); if (values[0]->second) { stats->IncrementCacheHitsCounter(); } else { stats->IncrementExecutedCounter(); } (*setter)(std::move(cfg)); }, [logger, repo_name, repo_type](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "repository {} of type {}:\n{}", nlohmann::json(repo_name).dump(), nlohmann::json(repo_type).dump(), msg), fatal); }); } /// \brief Perform checkout for an archive type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void ForeignFileCheckout( ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& foreign_file_git_map, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { auto foreign_file_repo_info = ParseForeignFileDescription(repo_desc, repo_name, logger); if (not foreign_file_repo_info) { return; } // get the WS root as git tree foreign_file_git_map->ConsumeAfterKeysReady( ts, {std::move(*foreign_file_repo_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = values[0]->first; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); if (values[0]->second) { stats->IncrementCacheHitsCounter(); } else { stats->IncrementExecutedCounter(); } (*setter)(std::move(cfg)); }, [logger, repo_name](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "foreign-file repository {}:\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); } /// \brief Perform checkout for a file type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void FileCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& fpath_git_map, bool fetch_absent, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { // enforce mandatory fields auto repo_desc_path = repo_desc->At("path"); if (not repo_desc_path) { (*logger)("FileCheckout: Mandatory field \"path\" is missing", /*fatal=*/true); return; } if (not repo_desc_path->get()->IsString()) { (*logger)(fmt::format("FileCheckout: Unsupported value {} for " "mandatory field \"path\"", repo_desc_path->get()->ToString()), /*fatal=*/true); return; } // get absolute path auto raw_fpath = repo_desc_path->get()->String(); if (raw_fpath.empty()) { raw_fpath = "."; } auto fpath = ToNormalPath(std::filesystem::absolute(raw_fpath)); // check "special" pragma auto repo_desc_pragma = repo_desc->At("pragma"); bool const& pragma_is_map = repo_desc_pragma and repo_desc_pragma->get()->IsMap(); auto pragma_special = pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = pragma_special and pragma_special->get()->IsString() and kPragmaSpecialMap.contains(pragma_special->get()->String()) ? std::make_optional( kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // check "to_git" pragma auto pragma_to_git = pragma_is_map ? repo_desc_pragma->get()->At("to_git") : std::nullopt; // resolving symlinks implies also to_git if (pragma_special_value == PragmaSpecial::ResolvePartially or pragma_special_value == PragmaSpecial::ResolveCompletely or (pragma_to_git and pragma_to_git->get()->IsBool() and pragma_to_git->get()->Bool())) { // check "absent" pragma auto pragma_absent = pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); // get the WS root as git tree FpathInfo fpath_info = { .fpath = fpath, .pragma_special = pragma_special_value, .absent = not fetch_absent and pragma_absent_value}; fpath_git_map->ConsumeAfterKeysReady( ts, {std::move(fpath_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = *values[0]; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); (*setter)(std::move(cfg)); // report work done stats->IncrementLocalPathsCounter(); }, [logger, repo_name](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "repository {} of type \"file\":\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); } else { // get the WS root as filesystem location nlohmann::json cfg({}); cfg["workspace_root"] = nlohmann::json::array({pragma_special_value == PragmaSpecial::Ignore ? FileRoot::kFileIgnoreSpecialMarker : "file", fpath.string()}); // explicit array SetReposTakeOver(&cfg, repos, repo_name); (*setter)(std::move(cfg)); // report local path stats->IncrementLocalPathsCounter(); } } /// \brief Perform checkout for a distdir type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void DistdirCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& distdir_git_map, bool fetch_absent, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { // enforce mandatory fields auto repo_desc_repositories = repo_desc->At("repositories"); if (not repo_desc_repositories) { (*logger)( "DistdirCheckout: Mandatory field \"repositories\" is missing", /*fatal=*/true); return; } if (not repo_desc_repositories->get()->IsList()) { (*logger)(fmt::format("DistdirCheckout: Unsupported value {} for " "mandatory field \"repositories\"", repo_desc_repositories->get()->ToString()), /*fatal=*/true); return; } // check "absent" pragma auto repo_desc_pragma = repo_desc->At("pragma"); auto pragma_absent = (repo_desc_pragma and repo_desc_pragma->get()->IsMap()) ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); // map of distfile to content auto distdir_content_for_id = std::make_shared< std::unordered_map>>(); auto distdir_content = std::make_shared>(); // get distdir list auto distdir_repos = repo_desc_repositories->get()->List(); // create list of archives to fetch auto dist_repos_to_fetch = std::make_shared>(); for (auto const& dist_repo : distdir_repos) { if (not dist_repo->IsString()) { (*logger)(fmt::format("DistdirCheckout: Unsupported value {} for " "\"repositories\" list entry", dist_repo->ToString()), /*fatal=*/true); return; } // get name of dist_repo auto dist_repo_name = dist_repo->String(); // check that repo name exists auto repos_dist_repo_name = repos->At(dist_repo_name); if (not repos_dist_repo_name) { (*logger)(fmt::format("DistdirCheckout: No repository named {}", nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } auto repo_desc = repos_dist_repo_name->get()->At("repository"); if (not repo_desc) { (*logger)( fmt::format("DistdirCheckout: Mandatory key \"repository\" " "missing for repository {}", nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc->get(), repos); if (not resolved_repo_desc) { (*logger)( fmt::format("DistdirCheckout: Found cyclic dependency for " "repository {}", nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } auto repo_type = (*resolved_repo_desc)->At("type"); if (not repo_type) { (*logger)( fmt::format("DistdirCheckout: Mandatory key \"type\" missing " "for repository {}", nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } if (not repo_type->get()->IsString()) { (*logger)(fmt::format("DistdirCheckout: Unsupported value {} for " "key \"type\" for repository {}", repo_type->get()->ToString(), nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } // get repo_type auto repo_type_str = repo_type->get()->String(); auto const checkout_type_it = kCheckoutTypeMap.find(repo_type_str); if (checkout_type_it == kCheckoutTypeMap.end()) { (*logger)(fmt::format("DistdirCheckout: Unknown type {} for " "repository {}", nlohmann::json(repo_type_str).dump(), nlohmann::json(dist_repo_name).dump()), /*fatal=*/true); return; } // only do work if repo is archive type if (checkout_type_it->second == CheckoutType::Archive) { auto const archive = ParseArchiveContent(*resolved_repo_desc, dist_repo_name); if (not archive) { (*logger)(fmt::format("DistdirCheckout: an error occurred " "while parsing repository {}\n{}", nlohmann::json(dist_repo_name).dump(), archive.error()), /*fatal=*/true); return; } // add to distdir content map auto repo_distfile = (archive->distfile ? archive->distfile.value() : std::filesystem::path(archive->fetch_url) .filename() .string()); distdir_content_for_id->insert_or_assign( repo_distfile, std::make_pair(archive->content_hash.Hash(), false)); distdir_content->insert_or_assign(repo_distfile, archive->content_hash.Hash()); // add to fetch list dist_repos_to_fetch->emplace_back(*archive); } } // get hash of distdir content auto distdir_content_id = HashFunction{HashFunction::Type::GitSHA1} .HashBlobData(nlohmann::json(*distdir_content_for_id).dump()) .HexString(); // get the WS root as git tree DistdirInfo distdir_info = { .content_id = distdir_content_id, .content_list = distdir_content, .repos_to_fetch = dist_repos_to_fetch, .origin = repo_name, .absent = not fetch_absent and pragma_absent_value}; distdir_git_map->ConsumeAfterKeysReady( ts, {std::move(distdir_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = values[0]->first; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); if (values[0]->second) { stats->IncrementCacheHitsCounter(); } else { stats->IncrementExecutedCounter(); } (*setter)(std::move(cfg)); }, [logger, repo_name](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "repository {} of type \"distdir\":\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); } /// \brief Perform checkout for a git tree type repository. /// Guarantees the logger is called exactly once with fatal if a failure occurs. void GitTreeCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& tree_id_git_map, bool fetch_absent, gsl::not_null const& stats, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { auto tree_info = ParseGitTree(repo_desc); if (not tree_info) { (*logger)( fmt::format("GitTreeCheckout: {}", std::move(tree_info).error()), /*fatal=*/true); return; } // check "special" pragma auto repo_desc_pragma = repo_desc->At("pragma"); bool const& pragma_is_map = repo_desc_pragma and repo_desc_pragma->get()->IsMap(); auto pragma_special = pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = pragma_special and pragma_special->get()->IsString() and kPragmaSpecialMap.contains(pragma_special->get()->String()) ? std::make_optional( kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // check "absent" pragma auto pragma_absent = pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); // populate struct TreeIdInfo tree_id_info = { .tree_info = *std::move(tree_info), .ignore_special = pragma_special_value == PragmaSpecial::Ignore, .absent = not fetch_absent and pragma_absent_value}; // get the WS root as git tree tree_id_git_map->ConsumeAfterKeysReady( ts, {std::move(tree_id_info)}, [repos = std::move(repos), repo_name, stats, setter]( auto const& values) { auto ws_root = values[0]->first; nlohmann::json cfg({}); cfg["workspace_root"] = ws_root; SetReposTakeOver(&cfg, repos, repo_name); if (values[0]->second) { stats->IncrementCacheHitsCounter(); } else { stats->IncrementExecutedCounter(); } (*setter)(std::move(cfg)); }, [logger, repo_name](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting the workspace root for " "repository {} of type \"git tree\":\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); } void PrecomputedRootCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::SubCallerPtr const& subcaller, ReposToSetupMap::LoggerPtr const& logger) { auto precomputed = ParsePrecomputedRoot(repo_desc); if (not precomputed) { (*logger)(fmt::format("Checkout of precomputed root {} failed:\n{}", nlohmann::json(repo_name).dump(), std::move(precomputed).error()), /*fatal=*/true); return; } std::string target_repo = precomputed->GetReferencedRepository(); (*subcaller)( {std::move(target_repo)}, [setter, repos = std::move(repos), repo_name, result = *std::move(precomputed)](auto const& /*unused*/) { nlohmann::json cfg{}; auto& ws_root = cfg["workspace_root"]; SetReposTakeOver(&cfg, repos, repo_name); bool absent = false; if (auto computed = result.AsComputed()) { ws_root.push_back(ComputedRoot::kMarker); ws_root.push_back(computed->repository); ws_root.push_back(computed->target_module); ws_root.push_back(computed->target_name); ws_root.push_back(computed->config); absent = computed->absent; } else if (auto tree_structure = result.AsTreeStructure()) { ws_root.push_back(TreeStructureRoot::kMarker); ws_root.push_back(tree_structure->repository); absent = tree_structure->absent; } if (absent) { auto pragma = nlohmann::json::object(); pragma["absent"] = true; ws_root.push_back(pragma); } std::invoke(*setter, std::move(cfg)); }, logger); } } // namespace auto CreateReposToSetupMap( std::shared_ptr const& config, std::optional const& main, bool interactive, gsl::not_null const& commit_git_map, gsl::not_null const& content_git_map, gsl::not_null const& foreign_file_git_map, gsl::not_null const& fpath_git_map, gsl::not_null const& distdir_git_map, gsl::not_null const& tree_id_git_map, bool fetch_absent, gsl::not_null const& stats, std::size_t jobs) -> ReposToSetupMap { auto setup_repo = [config, main, interactive, commit_git_map, content_git_map, foreign_file_git_map, fpath_git_map, distdir_git_map, tree_id_git_map, fetch_absent, stats](auto ts, auto setter, auto logger, auto subcaller, auto const& key) { auto repos = (*config)["repositories"]; if (main and (key == *main) and interactive) { // no repository checkout required nlohmann::json cfg({}); SetReposTakeOver(&cfg, repos, key); stats->IncrementLocalPathsCounter(); (*setter)(std::move(cfg)); } else { // repository requires checkout auto repo_desc_key = repos->At(key); if (not repo_desc_key) { (*logger)(fmt::format( "Config: Missing config entry for repository {}", nlohmann::json(key).dump()), /*fatal=*/true); return; } if (not repo_desc_key->get()->IsMap()) { (*logger)(fmt::format("Config: Config entry for repository {} " "is not a map", nlohmann::json(key).dump()), /*fatal=*/true); return; } auto repo_desc = repo_desc_key->get()->At("repository"); if (not repo_desc) { (*logger)(fmt::format("Config: Mandatory key \"repository\" " "missing for repository {}", nlohmann::json(key).dump()), /*fatal=*/true); return; } auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc->get(), repos); if (not resolved_repo_desc) { (*logger)(fmt::format("Config: Found cyclic dependency for " "repository {}", nlohmann::json(key).dump()), /*fatal=*/true); return; } if (not resolved_repo_desc.value()->IsMap()) { (*logger)(fmt::format("Config: Repository {} resolves to a " "non-map description", nlohmann::json(key).dump()), /*fatal=*/true); return; } auto repo_type = (*resolved_repo_desc)->At("type"); if (not repo_type) { (*logger)(fmt::format("Config: Mandatory key \"type\" missing " "for repository {}", nlohmann::json(key).dump()), /*fatal=*/true); return; } if (not repo_type->get()->IsString()) { (*logger)(fmt::format("Config: Unsupported value {} for key " "\"type\" for repository {}", repo_type->get()->ToString(), nlohmann::json(key).dump()), /*fatal=*/true); return; } // get repo_type auto repo_type_str = repo_type->get()->String(); auto const checkout_type_it = kCheckoutTypeMap.find(repo_type_str); if (checkout_type_it == kCheckoutTypeMap.end()) { (*logger)( fmt::format("Config: Unknown type {} for repository {}", nlohmann::json(repo_type_str).dump(), nlohmann::json(key).dump()), /*fatal=*/true); return; } // setup a wrapped_logger auto wrapped_logger = std::make_shared( [logger, repo_name = key](auto const& msg, bool fatal) { (*logger)(fmt::format("While setting up repository {}:\n{}", nlohmann::json(repo_name).dump(), msg), fatal); }); // do checkout switch (checkout_type_it->second) { case CheckoutType::Git: { GitCheckout(*resolved_repo_desc, std::move(repos), key, commit_git_map, stats, ts, setter, wrapped_logger); break; } case CheckoutType::Archive: { ArchiveCheckout(*resolved_repo_desc, std::move(repos), key, repo_type_str, content_git_map, stats, ts, setter, wrapped_logger); break; } case CheckoutType::ForeignFile: { ForeignFileCheckout(*resolved_repo_desc, std::move(repos), key, foreign_file_git_map, stats, ts, setter, wrapped_logger); break; } case CheckoutType::File: { FileCheckout(*resolved_repo_desc, std::move(repos), key, fpath_git_map, fetch_absent, stats, ts, setter, wrapped_logger); break; } case CheckoutType::Distdir: { DistdirCheckout(*resolved_repo_desc, std::move(repos), key, distdir_git_map, fetch_absent, stats, ts, setter, wrapped_logger); break; } case CheckoutType::GitTree: { GitTreeCheckout(*resolved_repo_desc, std::move(repos), key, tree_id_git_map, fetch_absent, stats, ts, setter, wrapped_logger); break; } case CheckoutType::Precomputed: { stats->IncrementComputedCounter(); PrecomputedRootCheckout(*resolved_repo_desc, std::move(repos), key, setter, subcaller, wrapped_logger); break; } } } }; return AsyncMapConsumer(setup_repo, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/repo_map/repos_to_setup_map.hpp000066400000000000000000000047471516554100600312050ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/root_maps/commit_git_map.hpp" #include "src/other_tools/root_maps/content_git_map.hpp" #include "src/other_tools/root_maps/distdir_git_map.hpp" #include "src/other_tools/root_maps/foreign_file_git_map.hpp" #include "src/other_tools/root_maps/fpath_git_map.hpp" #include "src/other_tools/root_maps/tree_id_git_map.hpp" /// \brief Maps a global repo name to a JSON object containing the workspace /// root and the TAKE_OVER fields. using ReposToSetupMap = AsyncMapConsumer; auto CreateReposToSetupMap( std::shared_ptr const& config, std::optional const& main, bool interactive, gsl::not_null const& commit_git_map, gsl::not_null const& content_git_map, gsl::not_null const& foreign_file_git_map, gsl::not_null const& fpath_git_map, gsl::not_null const& distdir_git_map, gsl::not_null const& tree_id_git_map, bool fetch_absent, gsl::not_null const& stats, std::size_t jobs) -> ReposToSetupMap; // use explicit cast to std::function to allow template deduction when used static const std::function kReposToSetupPrinter = [](std::string const& x) -> std::string { return x; }; #endif // INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/000077500000000000000000000000001516554100600247525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/TARGETS000066400000000000000000000205671516554100600260200ustar00rootroot00000000000000{ "distdir_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["distdir_git_map"] , "hdrs": ["distdir_git_map.hpp"] , "srcs": ["distdir_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "tmp_dir"] ] } , "commit_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["commit_git_map"] , "hdrs": ["commit_git_map.hpp"] , "srcs": ["commit_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "tmp_dir"] ] } , "fpath_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["fpath_git_map"] , "hdrs": ["fpath_git_map.hpp"] , "srcs": ["fpath_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/buildtool/file_system/symlinks", "resolve_symlinks_map"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path_hash"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "tmp_dir"] ] } , "content_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["content_git_map"] , "hdrs": ["content_git_map.hpp"] , "srcs": ["content_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/file_system/symlinks", "resolve_symlinks_map"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/archive", "archive_ops"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "tmp_dir"] ] } , "foreign_file_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["foreign_file_git_map"] , "hdrs": ["foreign_file_git_map.hpp"] , "srcs": ["foreign_file_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "git_cas"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "fs_utils"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "other_tools", "root_maps"] } , "tree_id_git_map": { "type": ["@", "rules", "CC", "library"] , "name": ["tree_id_git_map"] , "hdrs": ["tree_id_git_map.hpp"] , "srcs": ["tree_id_git_map.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "common"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] , ["src/other_tools/git_operations", "git_ops_types"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "tmp_dir"] ] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/commit_git_map.cpp000066400000000000000000001533131516554100600304540ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/commit_git_map.hpp" #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { [[nodiscard]] auto GitURLIsPath(std::string const& url) noexcept -> std::optional { static auto const kAbsPath = std::string{"/"}; static auto const kRelPath = std::string{"./"}; static auto const kFileScheme = std::string{"file://"}; if (url.starts_with(kAbsPath) or url.starts_with(kRelPath)) { return ToNormalPath(url).string(); } if (url.starts_with(kFileScheme)) { return ToNormalPath(url.substr(kFileScheme.length())).string(); } return std::nullopt; } [[nodiscard]] auto IsCacheGitRoot( StorageConfig const& native_storage_config, std::filesystem::path const& repo_root) noexcept -> bool { return std::filesystem::absolute(ToNormalPath(repo_root)) == std::filesystem::absolute( ToNormalPath(native_storage_config.GitRoot())); } /// \brief Helper function for ensuring the serve endpoint, if given, has the /// root if it was marked absent. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void EnsureRootAsAbsent(std::string const& tree_id, std::filesystem::path const& repo_root, GitRepoInfo const& repo_info, ServeApi const* serve, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // this is an absent root if (serve != nullptr) { // check if the serve endpoint has this root auto const has_tree = serve->CheckRootTree(tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve endpoint knows tree " "{} failed.", tree_id), /*fatal=*/true); return; } if (not *has_tree) { // try to see if serve endpoint has the information to prepare the // root itself auto const serve_result = serve->RetrieveTreeFromCommit(repo_info.hash, repo_info.subdir, /*sync_tree = */ false); if (serve_result) { // if serve has set up the tree, it must match what we expect auto const& served_tree_id = serve_result->tree; if (tree_id != served_tree_id) { (*logger)(fmt::format("Mismatch in served root tree " "id:\nexpected {}, but got {}", tree_id, served_tree_id), /*fatal=*/true); return; } } else { // check if serve failure was due to commit not being found or // it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)(fmt::format("Serve endpoint failed to set up " "root from known commit {}", repo_info.hash), /*fatal=*/true); return; } auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, tree_id, /*size_unknown=*/0, /*is_tree=*/true); if (not digest.has_value()) { (*logger)(std::move(digest).error(), /*fatal=*/true); return; } // the tree is known locally, so we can upload it to remote CAS // for the serve endpoint to retrieve it and set up the root auto uploaded = serve->UploadTree(*digest, repo_root); if (not uploaded.has_value()) { (*logger)(std::move(uploaded).error().Message(), /*fatal=*/true); return; } } } } else { // give warning (*logger)(fmt::format("Workspace root {} marked absent but no serve " "endpoint provided.", tree_id), /*fatal=*/false); } // set root as absent (*ws_setter)(std::pair( nlohmann::json::array({repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); } /// \brief Helper function for improved readability. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, std::string const& subdir, bool ignore_special, StorageConfig const& native_storage_config, GitCASPtr const& git_cas, std::filesystem::path const& tree_id_file, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // write association of the root tree in id file if (not StorageUtils::WriteTreeIDFile(tree_id_file, root_tree_id)) { (*logger)(fmt::format("Failed to write tree id {} to file {}", root_tree_id, tree_id_file.string()), /*fatal=*/true); return; } // extract the subdir tree auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open cache object database {}", native_storage_config.GitRoot().string()), /*fatal=*/true); return; } auto wrapped_logger = std::make_shared( [logger, subdir, tree = root_tree_id](auto const& msg, bool fatal) { (*logger)(fmt::format("While getting subdir {} in tree {}:\n{}", subdir, tree, msg), fatal); }); auto tree_id = git_repo->GetSubtreeFromTree(root_tree_id, subdir, wrapped_logger); if (not tree_id) { return; } // set the workspace root as present (*ws_setter)(std::pair( nlohmann::json::array({ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, native_storage_config.GitRoot().string()}), false)); } void TagAndSetRoot(std::filesystem::path const& repo_root, GitRepoInfo const& repo_info, bool finish_task, GitCASPtr const& git_cas, gsl::not_null const& critical_git_op_map, gsl::not_null const& progress, gsl::not_null const& ts, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { GitOpKey op_key = {.params = { repo_root, // target_path repo_info.hash, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TAG}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [git_cas, repo_info, repo_root, progress, ws_setter, logger, finish_task](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subtree from commit:\n{}", msg), fatal); }); // get tree id and return workspace root auto res = git_repo->GetSubtreeFromCommit( repo_info.hash, repo_info.subdir, wrapped_logger); if (not res) { return; } // set the workspace root as present if (finish_task) { progress->TaskTracker().Stop(repo_info.origin); } (*ws_setter)( std::pair(nlohmann::json::array( {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *std::move(res), // subtree id repo_root}), /*is_cache_hit=*/false)); }, [logger, target_path = repo_root](auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op KEEP_TAG " "for target {}:\n{}", target_path.string(), msg), fatal); }); } void TakeCommitFromOlderGeneration( std::filesystem::path const& source, gsl::not_null const& native_storage_config, std::filesystem::path const& repo_root, GitRepoInfo const& repo_info, GitCASPtr const& git_cas, gsl::not_null const& critical_git_op_map, gsl::not_null const& progress, gsl::not_null const& ts, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { GitOpKey op_key = {.params = { source, // target_path repo_info.hash, // git_hash "Tag commit for fetching" // message }, .op_type = GitOpType::KEEP_TAG}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [logger, git_cas, repo_root, native_storage_config, source, repo_info, critical_git_op_map, progress, ts, ws_setter](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } auto tag = *op_result.result; auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } auto fetch_logger = std::make_shared( [logger, tag, source](auto const& msg, bool fatal) { (*logger)(fmt::format("While fetching {} from {}:\n{}", tag, source.string(), msg), fatal); }); if (not git_repo->LocalFetchViaTmpRepo( *native_storage_config, source, tag, fetch_logger)) { return; } TagAndSetRoot(repo_root, repo_info, false, git_cas, critical_git_op_map, progress, ts, ws_setter, logger); }, [logger, source, hash = repo_info.hash](auto const& msg, bool fatal) { (*logger)(fmt::format("While tagging {} in {} for fetching:\n{}", source.string(), hash, msg), fatal); }); } void NetworkFetchAndSetPresentRoot( GitRepoInfo const& repo_info, std::filesystem::path const& repo_root, std::string const& fetch_repo, MirrorsPtr const& additional_mirrors, GitCASPtr const& git_cas, StorageConfig const& native_storage_config, gsl::not_null const& critical_git_op_map, std::string const& git_bin, std::vector const& launcher, bool fetch_absent, gsl::not_null const& progress, gsl::not_null const& ts, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // reaching here can only result in a root that is present if (repo_info.absent and not fetch_absent) { (*logger)( fmt::format("Cannot create workspace root as absent for commit {}.", repo_info.hash), /*fatal=*/true); return; } auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } // store failed attempts for subsequent logging bool fetched{false}; std::string err_messages{}; // keep all remotes checked to report them in case fetch fails std::string remotes_buffer{}; // try repo url auto all_mirrors = std::vector({fetch_repo}); // try repo mirrors afterwards all_mirrors.insert( all_mirrors.end(), repo_info.mirrors.begin(), repo_info.mirrors.end()); if (auto preferred_hostnames = MirrorsUtils::GetPreferredHostnames(additional_mirrors); not preferred_hostnames.empty()) { all_mirrors = MirrorsUtils::SortByHostname(all_mirrors, preferred_hostnames); } // always try local mirrors first auto local_mirrors = MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_repo); all_mirrors.insert( all_mirrors.begin(), local_mirrors.begin(), local_mirrors.end()); auto inherit_env = MirrorsUtils::GetInheritEnv(additional_mirrors, repo_info.inherit_env); for (auto mirror : all_mirrors) { auto mirror_path = GitURLIsPath(mirror); if (mirror_path) { mirror = std::filesystem::absolute(*mirror_path).string(); } auto wrapped_logger = std::make_shared( [mirror, &err_messages](auto const& msg, bool /*fatal*/) { err_messages += fmt::format( "While attempting fetch from URL {}:\n{}\n", mirror, msg); }); if (git_repo->FetchViaTmpRepo(native_storage_config, mirror, repo_info.branch, inherit_env, git_bin, launcher, wrapped_logger)) { fetched = true; break; } // add local mirror to buffer remotes_buffer.append(fmt::format("\n> {}", mirror)); } if (not fetched) { // log fetch failure and list the remotes tried (*logger)( fmt::format("While trying to fetch from provided remotes:\n{}Fetch " "failed for the provided remotes{}", err_messages, remotes_buffer), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While checking commit exists:\n{}", msg), fatal); }); // check if commit exists now, after fetch auto is_commit_present = git_repo->CheckCommitExists(repo_info.hash, wrapped_logger); if (not is_commit_present) { return; } if (not *is_commit_present) { // commit could not be fetched, so fail (*logger)(fmt::format( "Could not fetch commit {} from branch {} for remote {}", repo_info.hash, repo_info.branch, fetch_repo), /*fatal=*/true); return; } // if witnessing repository is the Git cache, then also tag the commit if (IsCacheGitRoot(native_storage_config, repo_root)) { TagAndSetRoot(repo_root, repo_info, true, git_cas, critical_git_op_map, progress, ts, ws_setter, logger); } else { auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subtree from commit:\n{}", msg), fatal); }); // get tree id and return workspace root auto res = git_repo->GetSubtreeFromCommit( repo_info.hash, repo_info.subdir, wrapped_logger); if (not res) { return; } // set the workspace root as present progress->TaskTracker().Stop(repo_info.origin); (*ws_setter)(std::pair( nlohmann::json::array({repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *std::move(res), // subtree id repo_root}), /*is_cache_hit=*/false)); } } /// \brief Contains the main logic for this async map. It ensures the commit is /// available for processing (including fetching for a present root) and setting /// the root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void EnsureCommit( GitRepoInfo const& repo_info, std::filesystem::path const& repo_root, std::string const& fetch_repo, MirrorsPtr const& additional_mirrors, GitCASPtr const& git_cas, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, std::string const& git_bin, std::vector const& launcher, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool fetch_absent, gsl::not_null const& progress, gsl::not_null const& ts, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // link fake repo to odb auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While checking commit exists:\n{}", msg), fatal); }); auto is_commit_present = git_repo->CheckCommitExists(repo_info.hash, wrapped_logger); if (is_commit_present == std::nullopt) { return; } if (not is_commit_present.value()) { auto tree_id_file = StorageUtils::GetCommitTreeIDFile( *native_storage_config, repo_info.hash); // Check if we have stored a file association between commit and tree; // if an association file exists, the respective tree MUST be in the // Git cache if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); if (not resolved_tree_id) { (*logger)(fmt::format("Failed to read tree id from file {}", tree_id_file.string()), /*fatal=*/true); return; } auto just_git_cas = GitCAS::Open(native_storage_config->GitRoot()); if (not just_git_cas) { (*logger)( fmt::format("Could not open Git cache database {}", native_storage_config->GitRoot().string()), /*fatal=*/true); return; } auto just_git_repo = GitRepo::Open(just_git_cas); if (not just_git_repo) { (*logger)( fmt::format("Could not open Git cache repository {}", native_storage_config->GitRoot().string()), /*fatal=*/true); return; } // extract the subdir tree wrapped_logger = std::make_shared( [logger, subdir = repo_info.subdir, tree = *resolved_tree_id]( auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subdir {} in tree {}:\n{}", subdir, tree, msg), fatal); }); auto tree_id = just_git_repo->GetSubtreeFromTree( *resolved_tree_id, repo_info.subdir, wrapped_logger); if (not tree_id) { return; } // set the workspace root if (repo_info.absent and not fetch_absent) { // try by all available means to generate & set the absent root EnsureRootAsAbsent( *tree_id, native_storage_config->GitRoot(), /*repo_root*/ repo_info, serve, ws_setter, logger); } else { // this root is present (*ws_setter)( std::pair(nlohmann::json::array( {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, native_storage_config->GitRoot().string()}), /*is_cache_hit=*/false)); } // done! return; } // Check older generations for presence of the commit for (std::size_t generation = 1; generation < native_storage_config->num_generations; generation++) { auto old = native_storage_config->GitGenerationRoot(generation); if (FileSystemManager::IsDirectory(old)) { auto old_repo = GitRepo::Open(old); auto no_logging = std::make_shared( [](auto /*unused*/, auto /*unused*/) {}); if (old_repo) { auto check_result = old_repo->CheckCommitExists(repo_info.hash, no_logging); if (check_result and *check_result) { TakeCommitFromOlderGeneration(old, native_storage_config, repo_root, repo_info, git_cas, critical_git_op_map, progress, ts, ws_setter, logger); return; } } } } // TODO(aehlig): should we also check for commit-tree association in // older generations // Not present locally, we have to fetch progress->TaskTracker().Start(repo_info.origin); // check if commit is known to remote serve service if (serve != nullptr) { // if root purely absent, request only the subdir tree if (repo_info.absent and not fetch_absent) { auto const serve_result = serve->RetrieveTreeFromCommit(repo_info.hash, repo_info.subdir, /*sync_tree = */ false); if (serve_result) { // set the workspace root as absent progress->TaskTracker().Stop(repo_info.origin); (*ws_setter)(std::pair( nlohmann::json::array( {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, serve_result->tree}), /*is_cache_hit=*/false)); return; } // check if serve failure was due to commit not being found or // it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)(fmt::format("Serve endpoint failed to set up " "root from known commit {}", repo_info.hash), /*fatal=*/true); return; } } // otherwise, request (and sync) the whole commit tree, to ensure // we maintain the id file association else { auto const serve_result = serve->RetrieveTreeFromCommit(repo_info.hash, /*subdir = */ ".", /*sync_tree = */ true); if (serve_result) { auto const root_tree_id = serve_result->tree; auto const remote_digest = serve_result->digest; // verify if we know the tree already in the local Git cache GitOpKey op_key = {.params = { native_storage_config ->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [root_tree_id, remote_digest, tree_id_file, repo_info, repo_root, fetch_repo, additional_mirrors, native_storage_config, git_cas, critical_git_op_map, import_to_git_map, git_bin, launcher, local_api, remote_api, fetch_absent, progress, ts, ws_setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } auto just_git_repo = GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { (*logger)( fmt::format("Could not open Git " "cache repository " "{}", native_storage_config->GitRoot() .string()), /*fatal=*/true); return; } // check tree existence auto wrapped_logger = std::make_shared( [logger, native_storage_config, tree = root_tree_id](auto const& msg, bool fatal) { (*logger)( fmt::format( "While verifying presence of " "tree {} in repository {}:\n{}", tree, native_storage_config->GitRoot() .string(), msg), fatal); }); auto tree_present = just_git_repo->CheckTreeExists( root_tree_id, wrapped_logger); if (not tree_present) { return; } if (*tree_present) { progress->TaskTracker().Stop(repo_info.origin); // write association to id file, get subdir // tree, and set the workspace root as present WriteIdFileAndSetWSRoot( root_tree_id, repo_info.subdir, repo_info.ignore_special, *native_storage_config, op_result.git_cas, tree_id_file, ws_setter, logger); return; } // now check if the tree is in the local checkout, // if this checkout is not our Git cache; this can // save an unnecessary remote CAS call if (not IsCacheGitRoot(*native_storage_config, repo_root)) { auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { (*logger)(fmt::format("Could not open Git " "repository {}", repo_root.string()), /*fatal=*/true); return; } // check tree existence wrapped_logger = std::make_shared( [logger, tree = root_tree_id, repo_root](auto const& msg, bool fatal) { (*logger)( fmt::format( "While verifying presence " "of tree {} in repository " "{}:\n{}", tree, repo_root.string(), msg), fatal); }); tree_present = git_repo->CheckTreeExists( root_tree_id, wrapped_logger); if (not tree_present) { return; } if (*tree_present) { progress->TaskTracker().Stop( repo_info.origin); // get subdir tree and set the workspace // root as present; as this tree is not in // our Git cache, no file association should // be stored wrapped_logger = std::make_shared< AsyncMapConsumerLogger>( [logger, subdir = repo_info.subdir, tree = root_tree_id](auto const& msg, bool fatal) { (*logger)( fmt::format( "While getting subdir {} " "in tree {}:\n{}", subdir, tree, msg), fatal); }); auto tree_id = git_repo->GetSubtreeFromTree( root_tree_id, repo_info.subdir, wrapped_logger); if (not tree_id) { return; } // set the workspace root as present (*ws_setter)(std::pair( nlohmann::json::array( {repo_info.ignore_special ? FileRoot:: kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, repo_root.string()}), false)); // done! return; } } // try to get root tree from remote CAS; use the // digest received from serve; whether native or // compatible, it will either way be imported to Git if (remote_api != nullptr and remote_digest and remote_api->RetrieveToCas( {Artifact::ObjectInfo{ .digest = *remote_digest, .type = ObjectType::Tree}}, *local_api)) { progress->TaskTracker().Stop(repo_info.origin); // Move tree from local CAS to local Git storage auto tmp_dir = native_storage_config->CreateTypedTmpDir( "fetch-absent-root"); if (not tmp_dir) { (*logger)( fmt::format( "Failed to create tmp directory " "after fetching root tree {} for " "absent commit {}", root_tree_id, repo_info.hash), /*fatal=*/true); return; } if (not local_api->RetrieveToPaths( {Artifact::ObjectInfo{ .digest = *remote_digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { (*logger)(fmt::format( "Failed to copy fetched root " "tree {} to {}", root_tree_id, tmp_dir->GetPath().string()), /*fatal=*/true); return; } CommitInfo c_info{ tmp_dir->GetPath(), "tree", root_tree_id}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive root_tree_id, native_storage_config, subdir = repo_info.subdir, ignore_special = repo_info.ignore_special, just_git_cas = op_result.git_cas, tree_id_file, ws_setter, logger](auto const& values) { if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // sanity check: we should get the // expected tree if (values[0]->first != root_tree_id) { (*logger)( fmt::format( "Mismatch in imported git " "tree id:\nexpected {}, " "but got {}", root_tree_id, values[0]->first), /*fatal=*/true); return; } // tree is now in Git cache; // write association to id file, get // subdir tree, and set the workspace // root as present WriteIdFileAndSetWSRoot( root_tree_id, subdir, ignore_special, *native_storage_config, just_git_cas, tree_id_file, ws_setter, logger); }, [logger, tmp_dir, root_tree_id]( auto const& msg, bool fatal) { (*logger)( fmt::format( "While moving root tree {} " "from {} to local git:\n{}", root_tree_id, tmp_dir->GetPath().string(), msg), fatal); }); return; } // just serve did not make the tree available in // the remote CAS, so fall back to network NetworkFetchAndSetPresentRoot( repo_info, repo_root, fetch_repo, additional_mirrors, git_cas, *native_storage_config, critical_git_op_map, git_bin, launcher, fetch_absent, progress, ts, ws_setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT bare for " "target {}:\n{}", target_path.string(), msg), fatal); }); // done! return; } // check if serve failure was due to commit not being found // or it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)(fmt::format("Serve endpoint failed to set up " "root from known commit {}", repo_info.hash), /*fatal=*/true); return; } } } NetworkFetchAndSetPresentRoot(repo_info, repo_root, fetch_repo, additional_mirrors, git_cas, *native_storage_config, critical_git_op_map, git_bin, launcher, fetch_absent, progress, ts, ws_setter, logger); } else { // commit is present in given repository progress->TaskTracker().Start(repo_info.origin); // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subtree from commit:\n{}", msg), fatal); }); // get tree id and return workspace root auto res = git_repo->GetSubtreeFromCommit( repo_info.hash, repo_info.subdir, wrapped_logger); if (not res) { return; } auto subtree = *std::move(res); // set the workspace root if (repo_info.absent and not fetch_absent) { // try by all available means to generate and set the absent root EnsureRootAsAbsent( subtree, repo_root, repo_info, serve, ws_setter, logger); } else { // set root as present (*ws_setter)( std::pair(nlohmann::json::array( {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, subtree, repo_root.string()}), /*is_cache_hit=*/true)); } } } } // namespace /// \brief Create a CommitGitMap object auto CreateCommitGitMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, LocalPathsPtr const& just_mr_paths, MirrorsPtr const& additional_mirrors, std::string const& git_bin, std::vector const& launcher, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool fetch_absent, gsl::not_null const& progress, std::size_t jobs) -> CommitGitMap { auto commit_to_git = [critical_git_op_map, import_to_git_map, just_mr_paths, additional_mirrors, git_bin, launcher, serve, native_storage_config, local_api, remote_api, fetch_absent, progress](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { // get root for repo (making sure that if repo is a path, it is // absolute) std::string fetch_repo = key.repo_url; auto fetch_repo_path = GitURLIsPath(fetch_repo); if (fetch_repo_path) { fetch_repo = std::filesystem::absolute(*fetch_repo_path).string(); } std::filesystem::path repo_root = StorageUtils::GetGitRoot( *native_storage_config, just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { .params = { repo_root, // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path not just_mr_paths->git_checkout_locations.contains( fetch_repo) // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, repo_root, fetch_repo, additional_mirrors, critical_git_op_map, import_to_git_map, git_bin, launcher, serve, native_storage_config, local_api, remote_api, fetch_absent, progress, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } // setup a wrapped_logger auto wrapped_logger = std::make_shared( [logger, target_path = repo_root](auto const& msg, bool fatal) { (*logger)(fmt::format("While ensuring commit for " "repository {}:\n{}", target_path.string(), msg), fatal); }); EnsureCommit(key, repo_root, fetch_repo, additional_mirrors, op_result.git_cas, critical_git_op_map, import_to_git_map, git_bin, launcher, serve, native_storage_config, local_api, remote_api, fetch_absent, progress, ts, setter, wrapped_logger); }, [logger, target_path = repo_root](auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); }; return AsyncMapConsumer>( commit_to_git, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/commit_git_map.hpp000066400000000000000000000067741516554100600304710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" struct GitRepoInfo { // hash can be a commit or tree std::string hash; /* key */ std::string repo_url; std::string branch; std::string subdir; /* key */ std::vector inherit_env; std::vector mirrors; // name of repository for which work is done; used in progress reporting std::string origin; // create root that ignores symlinks bool ignore_special{}; /* key */ // create an absent root bool absent{}; /* key */ [[nodiscard]] auto operator==(const GitRepoInfo& other) const -> bool { return hash == other.hash and subdir == other.subdir and ignore_special == other.ignore_special and absent == other.absent; } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const GitRepoInfo& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.hash); hash_combine(&seed, ct.subdir); hash_combine(&seed, ct.ignore_special); hash_combine(&seed, ct.absent); return seed; } }; } // namespace std /// \brief Maps a Git repository commit hash to its tree workspace root, /// together with the information whether it was a cache hit. using CommitGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateCommitGitMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, LocalPathsPtr const& just_mr_paths, MirrorsPtr const& additional_mirrors, std::string const& git_bin, std::vector const& launcher, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, bool fetch_absent, gsl::not_null const& progress, std::size_t jobs) -> CommitGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/content_git_map.cpp000066400000000000000000001447631516554100600306470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/content_git_map.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/archive/archive_ops.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { /// \brief Extracts the archive of given type into the destination directory /// provided. Returns nullopt on success, or error string on failure. [[nodiscard]] auto ExtractArchive(std::filesystem::path const& archive, std::string const& repo_type, std::filesystem::path const& dst_dir) noexcept -> std::optional { if (repo_type == "archive") { return ArchiveOps::ExtractArchive( ArchiveType::TarAuto, archive, dst_dir); } if (repo_type == "zip") { return ArchiveOps::ExtractArchive( ArchiveType::ZipAuto, archive, dst_dir); } return "unrecognized repository type"; } /// \brief Helper function for ensuring the serve endpoint, if given, has the /// root if it was marked absent. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void EnsureRootAsAbsent( std::string const& tree_id, ArchiveRepoInfo const& key, ServeApi const* serve, gsl::not_null const& native_storage_config, bool is_cache_hit, ContentGitMap::SetterPtr const& ws_setter, ContentGitMap::LoggerPtr const& logger) { // this is an absent root if (serve != nullptr) { // check if the serve endpoint has this root auto const has_tree = serve->CheckRootTree(tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve endpoint knows tree " "{} failed.", tree_id), /*fatal=*/true); return; } if (not *has_tree) { // try to see if serve endpoint has the information to prepare the // root itself; this is redundant if root is not already cached bool on_serve = false; if (is_cache_hit) { auto const serve_result = serve->RetrieveTreeFromArchive( key.archive.content_hash.Hash(), key.repo_type, key.subdir, key.pragma_special, /*sync_tree=*/false); if (serve_result) { // if serve has set up the tree, it must match what we // expect if (tree_id != serve_result->tree) { (*logger)(fmt::format("Mismatch in served root tree " "id:\nexpected {}, but got {}", tree_id, serve_result->tree), /*fatal=*/true); return; } on_serve = true; } if (not serve_result.has_value() and serve_result.error() == GitLookupError::Fatal) { (*logger)(fmt::format("Serve endpoint failed to set up " "root from known archive content {}", key.archive.content_hash.Hash()), /*fatal=*/true); return; } } if (not on_serve) { // the tree is known locally, so we can upload it to remote CAS // for the serve endpoint to retrieve it and set up the root auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, tree_id, /*size_unknown=*/0, /*is_tree=*/true); if (not digest.has_value()) { (*logger)(std::move(digest).error(), /*fatal=*/true); return; } // the tree is known locally, so we can upload it to remote // CAS for the serve endpoint to retrieve it and set up the // root auto uploaded = serve->UploadTree( *digest, native_storage_config->GitRoot()); if (not uploaded.has_value()) { (*logger)(std::move(uploaded).error().Message(), /*fatal=*/true); return; } } } } else { // give warning (*logger)(fmt::format("Workspace root {} marked absent but no suitable " "serve endpoint provided.", tree_id), /*fatal=*/false); } // set root as absent (*ws_setter)( std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/is_cache_hit)); } /// \brief Called to get the resolved root (with respect to symlinks) from /// an unresolved tree. It guarantees the logger is called exactly once with /// fatal on failure, and the setter on success. void ResolveContentTree( ArchiveRepoInfo const& key, std::string const& tree_hash, GitCASPtr const& just_git_cas, bool is_cache_hit, bool is_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& critical_git_op_map, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& ws_setter, ContentGitMap::LoggerPtr const& logger) { if (key.pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( *native_storage_config, tree_hash, *key.pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); if (not resolved_tree_id) { (*logger)(fmt::format("Failed to read resolved tree id " "from file {}", tree_id_file.string()), /*fatal=*/true); return; } // set the workspace root if (is_absent) { EnsureRootAsAbsent(*resolved_tree_id, key, serve, native_storage_config, is_cache_hit, ws_setter, logger); } else { (*ws_setter)( std::pair(nlohmann::json::array( {FileRoot::kGitTreeMarker, *resolved_tree_id, native_storage_config->GitRoot().string()}), /*is_cache_hit=*/is_cache_hit)); } } else { // resolve tree; both source and target repos are the Git cache resolve_symlinks_map->ConsumeAfterKeysReady( ts, {GitObjectToResolve(tree_hash, ".", *key.pragma_special, /*known_info=*/std::nullopt, just_git_cas, just_git_cas)}, [critical_git_op_map, tree_hash, tree_id_file, is_cache_hit, key, is_absent, serve, native_storage_config, ts, ws_setter, logger](auto const& hashes) { auto const& resolved_tree_id = hashes[0]->id; // keep tree alive in Git cache via a tagged commit GitOpKey op_key = { .params = { native_storage_config ->GitRoot(), // target_path resolved_tree_id, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [resolved_tree_id, key, tree_id_file, is_absent, serve, native_storage_config, is_cache_hit, ws_setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tree failed", /*fatal=*/true); return; } // cache the resolved tree in the CAS map if (not StorageUtils::WriteTreeIDFile( tree_id_file, resolved_tree_id)) { (*logger)( fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), /*fatal=*/true); return; } // set the workspace root if (is_absent) { EnsureRootAsAbsent(resolved_tree_id, key, serve, native_storage_config, is_cache_hit, ws_setter, logger); } else { (*ws_setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, resolved_tree_id, native_storage_config->GitRoot() .string()}), /*is_cache_hit=*/is_cache_hit)); } }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "KEEP_TREE for target {}:\n{}", target_path.string(), msg), fatal); }); }, [logger, hash = key.archive.content_hash.Hash()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While resolving symlinks for " "content {}:\n{}", hash, msg), fatal); }); } } else { // set the workspace root as-is if (is_absent) { EnsureRootAsAbsent(tree_hash, key, serve, native_storage_config, is_cache_hit, ws_setter, logger); } else { (*ws_setter)( std::pair(nlohmann::json::array( {FileRoot::kGitTreeMarker, tree_hash, native_storage_config->GitRoot().string()}), /*is_cache_hit=*/is_cache_hit)); } } } /// \brief Called to store the file association and then set the root. /// It guarantees the logger is called exactly once with fatal on failure, /// and the setter on success. void WriteIdFileAndSetWSRoot( ArchiveRepoInfo const& key, std::string const& archive_tree_id, GitCASPtr const& just_git_cas, std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& critical_git_op_map, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // write to tree id file if (not StorageUtils::WriteTreeIDFile(archive_tree_id_file, archive_tree_id)) { (*logger)(fmt::format("Failed to write tree id to file {}", archive_tree_id_file.string()), /*fatal=*/true); return; } // we look for subtree in Git cache auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)("Could not open Git cache repository!", /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [&logger, subdir = key.subdir, tree = archive_tree_id](auto const& msg, bool fatal) { (*logger)(fmt::format("While getting subdir {} from tree {}:\n{}", subdir, tree, msg), fatal); }); // get subtree id auto subtree_hash = just_git_repo->GetSubtreeFromTree( archive_tree_id, key.subdir, wrapped_logger); if (not subtree_hash) { return; } // resolve tree and set workspace root ResolveContentTree(key, *subtree_hash, just_git_cas, false, /*is_cache_hit*/ is_absent, serve, native_storage_config, critical_git_op_map, resolve_symlinks_map, ts, setter, logger); } /// \brief Called when archive is in local CAS. Performs the import-to-git /// and follow-up processing. It guarantees the logger is called exactly /// once with fatal on failure, and the setter on success. void ExtractAndImportToGit( ArchiveRepoInfo const& key, std::filesystem::path const& content_cas_path, std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // extract archive auto tmp_dir = native_storage_config->CreateTypedTmpDir(key.repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for {} target {}", key.repo_type, key.archive.content_hash.Hash()), /*fatal=*/true); return; } auto res = ExtractArchive(content_cas_path, key.repo_type, tmp_dir->GetPath()); if (res != std::nullopt) { (*logger)(fmt::format("Failed to extract archive {} from CAS with " "error:\n{}", content_cas_path.string(), *res), /*fatal=*/true); return; } // import to git CommitInfo c_info{ tmp_dir->GetPath(), key.repo_type, key.archive.content_hash.Hash()}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive archive_tree_id_file, key, is_absent, serve, native_storage_config, critical_git_op_map, resolve_symlinks_map, ts, setter, logger](auto const& values) { // check for errors if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // only tree id is needed std::string archive_tree_id = values[0]->first; // write to id file and process subdir tree WriteIdFileAndSetWSRoot(key, archive_tree_id, values[0]->second, /*just_git_cas*/ archive_tree_id_file, is_absent, serve, native_storage_config, critical_git_op_map, resolve_symlinks_map, ts, setter, logger); }, [logger, target_path = tmp_dir->GetPath()](auto const& msg, bool fatal) { (*logger)(fmt::format("While importing target {} to Git:\n{}", target_path.string(), msg), fatal); }); } auto IdFileExistsInOlderGeneration( gsl::not_null const& native_storage_config, ArchiveRepoInfo const& key) -> std::optional { for (std::size_t generation = 1; generation < native_storage_config->num_generations; generation++) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile(*native_storage_config, key.repo_type, key.archive.content_hash.Hash(), generation); if (FileSystemManager::Exists(archive_tree_id_file)) { return generation; } } return std::nullopt; } void HandleLocallyKnownTree( ArchiveRepoInfo const& key, std::filesystem::path const& archive_tree_id_file, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // read archive_tree_id from file tree_id_file auto archive_tree_id = FileSystemManager::ReadFile(archive_tree_id_file); if (not archive_tree_id) { (*logger)(fmt::format("Failed to read tree id from file {}", archive_tree_id_file.string()), /*fatal=*/true); return; } // ensure Git cache // define Git operation to be done GitOpKey op_key = {.params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [archive_tree_id = *archive_tree_id, key, fetch_absent, serve, native_storage_config, critical_git_op_map, resolve_symlinks_map, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } // open fake repo wrap for GitCAS auto just_git_repo = GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { (*logger)("Could not open Git cache repository!", /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [&logger](auto const& msg, bool fatal) { (*logger)(fmt::format("While getting subtree from " "tree:\n{}", msg), fatal); }); // get subtree id auto subtree_hash = just_git_repo->GetSubtreeFromTree( archive_tree_id, key.subdir, wrapped_logger); if (not subtree_hash) { return; } // resolve tree and set workspace root (present or absent) ResolveContentTree( key, *subtree_hash, op_result.git_cas, /*is_cache_hit = */ true, /*is_absent = */ (key.absent and not fetch_absent), serve, native_storage_config, critical_git_op_map, resolve_symlinks_map, ts, setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT for " "target {}:\n{}", target_path.string(), msg), fatal); }); } void HandleKnownInOlderGenerationAfterImport( ArchiveRepoInfo const& key, const std::string& tree_id, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // Now that we have the tree persisted in the git repository of the youngest // generation; hence we can write the map-entry. auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( *native_storage_config, key.repo_type, key.archive.content_hash.Hash()); if (not StorageUtils::WriteTreeIDFile(archive_tree_id_file, tree_id)) { (*logger)(fmt::format("Failed to write tree id to file {}", archive_tree_id_file.string()), /*fatal=*/true); return; } // Now that we also have the the ID-file written, we're in the situation // as if we had a cache hit in the first place. HandleLocallyKnownTree(key, archive_tree_id_file, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); } void HandleKnownInOlderGenerationAfterTaggingAndInit( ArchiveRepoInfo const& key, std::string tree_id, std::string tag, GitCASPtr const& git_cas, std::filesystem::path const& source, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { (*logger)("Could not open just initialized repository {}", /*fatal=*/true); return; } auto fetch_logger = std::make_shared( [logger, tag, source](auto const& msg, bool fatal) { (*logger)(fmt::format("While fetching {} from {}:\n{}", tag, source.string(), msg), fatal); }); if (not git_repo->LocalFetchViaTmpRepo( *native_storage_config, source, tag, fetch_logger)) { return; } GitOpKey op_key = {.params = { native_storage_config->GitRoot(), // target_path tree_id, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, tree_id, git_cas, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } HandleKnownInOlderGenerationAfterImport(key, tree_id, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); }, [logger, tree_id](auto const& msg, bool fatal) { (*logger)( fmt::format( "While tagging to keep tree {} alive:\n{}", tree_id, msg), fatal); }); } void HandleKnownInOlderGenerationAfterTagging( ArchiveRepoInfo const& key, const std::string& tree_id, const std::string& tag, std::filesystem::path const& source, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { GitOpKey op_key = {.params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, tree_id, tag, source, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } HandleKnownInOlderGenerationAfterTaggingAndInit( key, tree_id, tag, op_result.git_cas, source, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); } void HandleKnownInOlderGeneration( ArchiveRepoInfo const& key, std::size_t generation, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile(*native_storage_config, key.repo_type, key.archive.content_hash.Hash(), generation); auto archive_tree_id = FileSystemManager::ReadFile(archive_tree_id_file); if (not archive_tree_id) { (*logger)(fmt::format("Failed to read tree id from file {}", archive_tree_id_file.string()), /*fatal=*/true); return; } auto source = native_storage_config->GitGenerationRoot(generation); GitOpKey op_key = {.params = { source, // target_path *archive_tree_id, // git_hash "Tag commit for fetching" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, tree_id = *archive_tree_id, source, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Keep tag failed", /*fatal=*/true); return; } auto tag = *op_result.result; HandleKnownInOlderGenerationAfterTagging(key, tree_id, tag, source, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); }, [logger, source, hash = *archive_tree_id](auto const& msg, bool fatal) { (*logger)( fmt::format("While tagging tree {} in {} for fetching:\n{}", source.string(), hash, msg), fatal); }); } } // namespace auto CreateContentGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, LocalPathsPtr const& just_mr_paths, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& native_storage, bool fetch_absent, gsl::not_null const& progress, std::size_t jobs) -> ContentGitMap { auto gitify_content = [content_cas_map, import_to_git_map, resolve_symlinks_map, critical_git_op_map, just_mr_paths, serve, native_storage_config, native_storage, fetch_absent, progress](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile(*native_storage_config, key.repo_type, key.archive.content_hash.Hash()); if (FileSystemManager::Exists(archive_tree_id_file)) { HandleLocallyKnownTree(key, archive_tree_id_file, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); } else if (auto generation = IdFileExistsInOlderGeneration( native_storage_config, key)) { HandleKnownInOlderGeneration(key, *generation, fetch_absent, serve, native_storage_config, resolve_symlinks_map, critical_git_op_map, ts, setter, logger); } else { // separate logic between absent and present roots if (key.absent and not fetch_absent) { // request the resolved subdir tree from the serve endpoint, if // given if (serve != nullptr) { auto const serve_result = serve->RetrieveTreeFromArchive( key.archive.content_hash.Hash(), key.repo_type, key.subdir, key.pragma_special, /*sync_tree = */ false); if (serve_result) { // set the workspace root as absent progress->TaskTracker().Stop(key.archive.origin); (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, serve_result->tree}), /*is_cache_hit = */ false)); return; } // check if serve failure was due to archive content // not being found or it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)( fmt::format("Serve endpoint failed to set up root " "from known archive content {}", key.archive.content_hash.Hash()), /*fatal=*/true); return; } } // if serve endpoint cannot set up the root, we might still be // able to set up the absent root with local information, and if // a serve endpoint exists we can upload it the root ourselves; // check if content already in CAS auto const& native_cas = native_storage->CAS(); auto const digest = ArtifactDigest{key.archive.content_hash, 0}; if (auto content_cas_path = native_cas.BlobPath(digest, /*is_executable=*/false)) { ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent = */ true, serve, native_storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, ts, setter, logger); // done return; } // check if content is in Git cache; // ensure Git cache GitOpKey op_key = { .params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [key, digest, archive_tree_id_file, critical_git_op_map, import_to_git_map, resolve_symlinks_map, just_mr_paths, serve, native_storage_config, native_storage, progress, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } auto const just_git_cas = op_result.git_cas; // open fake repo wrap for GitCAS auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)("Could not open Git cache repository!", /*fatal=*/true); return; } // verify if local Git knows content blob auto wrapped_logger = std::make_shared( [&logger, hash = key.archive.content_hash.Hash()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While verifying presence " "of blob {}:\n{}", hash, msg), fatal); }); auto res = just_git_repo->TryReadBlob( key.archive.content_hash.Hash(), wrapped_logger); if (not res.first) { // blob check failed return; } auto const& native_cas = native_storage->CAS(); if (res.second) { // blob found; add it to CAS if (not native_cas.StoreBlob( *res.second, /*is_executable=*/false)) { (*logger)(fmt::format( "Failed to store content " "{} to local CAS", key.archive.content_hash.Hash()), /*fatal=*/true); return; } if (auto content_cas_path = native_cas.BlobPath( digest, /*is_executable=*/false)) { ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent=*/true, serve, native_storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, ts, setter, logger); // done return; } // this should normally never be reached unless // something went really wrong (*logger)(fmt::format("Failed to retrieve blob {} " "from local CAS", digest.hash()), /*fatal=*/true); return; } progress->TaskTracker().Start(key.archive.origin); // add distfile to CAS auto repo_distfile = (key.archive.distfile ? key.archive.distfile.value() : std::filesystem::path(key.archive.fetch_url) .filename() .string()); StorageUtils::AddDistfileToCAS( *native_storage, repo_distfile, just_mr_paths); // check if content is in CAS now if (auto content_cas_path = native_cas.BlobPath( digest, /*is_executable=*/false)) { progress->TaskTracker().Stop(key.archive.origin); ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent=*/true, serve, native_storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, ts, setter, logger); // done return; } // report not being able to set up this root as absent (*logger)(fmt::format("Cannot create workspace root as " "absent for content {}.", key.archive.content_hash.Hash()), /*fatal=*/true); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); } else { // for a present root we need the archive to be present too content_cas_map->ConsumeAfterKeysReady( ts, {key.archive}, [archive_tree_id_file, key, native_storage_config, native_storage, critical_git_op_map, import_to_git_map, resolve_symlinks_map, ts, setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now auto const& native_cas = native_storage->CAS(); auto content_cas_path = native_cas .BlobPath( ArtifactDigest{key.archive.content_hash, 0}, /*is_executable=*/false) .value(); // root can only be present, so default all arguments // that refer to a serve endpoint ExtractAndImportToGit(key, content_cas_path, archive_tree_id_file, /*is_absent=*/false, /*serve=*/nullptr, native_storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, ts, setter, logger); }, [logger, hash = key.archive.content_hash.Hash()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While ensuring content {} is in " "CAS:\n{}", hash, msg), fatal); }); } } }; return AsyncMapConsumer>( gitify_content, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/content_git_map.hpp000066400000000000000000000044621516554100600306430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" /// \brief Maps the content of an archive to the resulting Git tree WS root, /// together with the information whether it was a cache hit. using ContentGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateContentGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, LocalPathsPtr const& just_mr_paths, gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& native_storage, bool fetch_absent, gsl::not_null const& progress, std::size_t jobs) -> ContentGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/distdir_git_map.cpp000066400000000000000000000657641516554100600306420ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/distdir_git_map.hpp" #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { /// \brief Create links from CAS content to distdir tmp directory [[nodiscard]] auto LinkToCAS( Storage const& storage, std::shared_ptr> const& content_list, std::filesystem::path const& tmp_dir) noexcept -> bool { auto const& cas = storage.CAS(); return std::all_of( content_list->begin(), content_list->end(), [&cas, tmp_dir](auto const& kv) { auto const digest = ArtifactDigestFactory::Create(cas.GetHashFunction().GetType(), kv.second, 0, /*is_tree=*/false); if (not digest) { return false; } auto content_path = cas.BlobPath(*digest, /*is_executable=*/false); if (content_path) { return FileSystemManager::CreateFileHardlink( *content_path, // from: cas_path/content_id tmp_dir / kv.first) // to: tmp_dir/name .has_value(); } return false; }); } /// \brief Called once we know we have the content blobs in local CAS in order /// to do the import-to-git step. Then it also sets the root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void ImportFromCASAndSetRoot( DistdirInfo const& key, StorageConfig const& native_storage_config, Storage const& native_storage, std::filesystem::path const& distdir_tree_id_file, gsl::not_null const& import_to_git_map, gsl::not_null const& ts, DistdirGitMap::SetterPtr const& setter, DistdirGitMap::LoggerPtr const& logger) { // create the links to CAS auto tmp_dir = native_storage_config.CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", key.content_id), /*fatal=*/true); return; } // link content from CAS into tmp dir if (not LinkToCAS(native_storage, key.content_list, tmp_dir->GetPath())) { (*logger)(fmt::format("Failed to create links to CAS content!", key.content_id), /*fatal=*/true); return; } // do import to git CommitInfo c_info{tmp_dir->GetPath(), "distdir", key.content_id}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, git_root = native_storage_config.GitRoot().string(), setter, logger](auto const& values) { // check for errors if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // only the tree is of interest std::string distdir_tree_id = values[0]->first; // write to tree id file if (not StorageUtils::WriteTreeIDFile(distdir_tree_id_file, distdir_tree_id)) { (*logger)(fmt::format("Failed to write tree id to file {}", distdir_tree_id_file.string()), /*fatal=*/true); return; } // set the workspace root as present (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, distdir_tree_id, git_root}), /*is_cache_hit=*/false)); }, [logger, target_path = tmp_dir->GetPath()](auto const& msg, bool fatal) { (*logger)(fmt::format("While importing target {} to git:\n{}", target_path.string(), msg), fatal); }); } } // namespace auto CreateDistdirGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& native_storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap { auto distdir_to_git = [content_cas_map, import_to_git_map, critical_git_op_map, serve, native_storage_config, native_storage, local_api, remote_api](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile( *native_storage_config, key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = FileSystemManager::ReadFile(distdir_tree_id_file); if (not distdir_tree_id) { (*logger)(fmt::format("Failed to read tree id from file {}", distdir_tree_id_file.string()), /*fatal=*/true); return; } // ensure Git cache // define Git operation to be done GitOpKey op_key = { .params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [distdir_tree_id = *distdir_tree_id, content_id = key.content_id, key, serve, native_storage_config, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } // subdir is "." here, so no need to deal with the Git cache // and we can simply set the workspace root if (key.absent) { if (serve != nullptr) { // check if serve endpoint has this root auto const has_tree = serve->CheckRootTree(distdir_tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve " "endpoint knows tree " "{} failed.", distdir_tree_id), /*fatal=*/true); return; } if (not *has_tree) { // try to see if serve endpoint has the // information to prepare the root itself auto const serve_result = serve->RetrieveTreeFromDistdir( key.content_list, /*sync_tree=*/false); if (serve_result) { // if serve has set up the tree, it must // match what we expect if (distdir_tree_id != serve_result->tree) { (*logger)( fmt::format( "Mismatch in served root tree " "id:\nexpected {}, but got {}", distdir_tree_id, serve_result->tree), /*fatal=*/true); return; } } else { // check if serve failure was due to distdir // content not being found or it is // otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)( fmt::format( "Serve endpoint failed to set " "up root from known distdir " "content {}", content_id), /*fatal=*/true); return; } auto digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, distdir_tree_id, /*size_unknown=*/0, /*is_tree=*/true); if (not digest.has_value()) { (*logger)(std::move(digest).error(), /*fatal=*/true); return; } // the tree is known locally, so we upload // it to remote CAS for the serve endpoint // to retrieve it and set up the root auto uploaded = serve->UploadTree( *digest, native_storage_config->GitRoot()); if (not uploaded.has_value()) { (*logger)(std::move(uploaded) .error() .Message(), /*fatal=*/true); return; } } } } else { // give warning (*logger)( fmt::format("Workspace root {} marked absent " "but no serve endpoint provided.", distdir_tree_id), /*fatal=*/false); } // set root as absent (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, distdir_tree_id}), /*is_cache_hit=*/true)); } else { // set root as present (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, distdir_tree_id, native_storage_config->GitRoot().string()}), /*is_cache_hit=*/true)); } }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); } // if no association file exists else { // create in-memory Git tree of distdir content to get the tree id GitRepo::tree_entries_t entries{}; entries.reserve(key.content_list->size()); for (auto const& kv : *key.content_list) { // tree_entries_t type expects raw ids auto raw_id = FromHexString(kv.second); if (not raw_id) { (*logger)(fmt::format("While processing distdir {}: " "Unexpected failure in conversion to " "raw id of distfile content {}", key.content_id, kv.second), /*is_fatal=*/true); return; } entries[*raw_id].emplace_back(kv.first, ObjectType::File); } auto tree = GitRepo::CreateShallowTree(entries); if (not tree) { (*logger)(fmt::format("Failed to construct in-memory tree for " "distdir content {}", key.content_id), /*is_fatal=*/true); return; } // get hash from raw_id auto const tree_id = ToHexString(tree->first); // get digest object auto const digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, tree_id, 0, /*is_tree=*/true); // use this knowledge of the resulting tree identifier to try to set // up the absent root without actually checking the local status of // each content blob individually if (key.absent) { if (serve != nullptr) { // first check if serve endpoint has tree auto const has_tree = serve->CheckRootTree(tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve " "endpoint knows tree " "{} failed.", tree_id), /*fatal=*/true); return; } if (*has_tree) { // set workspace root as absent (*setter)( std::pair(nlohmann::json::array( {FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } // try to see if serve endpoint has the information to // prepare the root itself auto const serve_result = serve->RetrieveTreeFromDistdir(key.content_list, /*sync_tree=*/false); if (serve_result) { // if serve has set up the tree, it must match what we // expect if (tree_id != serve_result->tree) { (*logger)( fmt::format("Mismatch in served root tree " "id:\nexpected {}, but got {}", tree_id, serve_result->tree), /*fatal=*/true); return; } // set workspace root as absent (*setter)( std::pair(nlohmann::json::array( {FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } // check if serve failure was due to distdir content not // being found or it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)( fmt::format("Serve endpoint failed to set up root " "from known distdir content {}", key.content_id), /*fatal=*/true); return; } // we cannot continue without a suitable remote set up if (remote_api == nullptr) { (*logger)(fmt::format( "Cannot create workspace root {} as " "absent for the provided serve endpoint.", tree_id), /*fatal=*/true); return; } // try to supply the serve endpoint with the tree via the // remote CAS if (digest and remote_api->IsAvailable(*digest)) { // tell serve to set up the root from the remote CAS // tree if (serve->GetTreeFromRemote(*digest)) { // set workspace root as absent (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } (*logger)(fmt::format("Serve endpoint failed to create " "workspace root {} that locally " "was marked absent.", tree_id), /*fatal=*/true); return; } // check if we have the tree in local CAS; if yes, upload it // to remote for the serve endpoint to find it if (digest and local_api->IsAvailable(*digest)) { if (not local_api->RetrieveToCas( {Artifact::ObjectInfo{ .digest = *digest, .type = ObjectType::Tree}}, *remote_api)) { (*logger)(fmt::format("Failed to sync tree {} from " "local CAS with remote CAS.", tree_id), /*fatal=*/true); return; } // tell serve to set up the root from the remote CAS // tree if (serve->GetTreeFromRemote(*digest)) { // set workspace root as absent (*setter)(std::pair( nlohmann::json::array( {FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } } // cannot create absent root with given information (*logger)( fmt::format("Serve endpoint failed to create workspace " "root {} that locally was marked absent.", tree_id), /*fatal=*/true); return; } // give warning (*logger)(fmt::format("Workspace root {} marked absent but no " "serve endpoint provided.", tree_id), /*fatal=*/false); // set workspace root as absent (*setter)(std::pair( nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}), false /*no cache hit*/)); return; } // if the root is not-absent, the order of checks is different; // first, look in the local CAS if (digest and local_api->IsAvailable(*digest)) { ImportFromCASAndSetRoot(key, *native_storage_config, *native_storage, distdir_tree_id_file, import_to_git_map, ts, setter, logger); // done return; } // now ask serve endpoint if it can set up the root; as this is for // a present root, a corresponding remote endpoint is needed if (serve != nullptr and remote_api != nullptr) { auto const serve_result = serve->RetrieveTreeFromDistdir(key.content_list, /*sync_tree=*/true); if (serve_result) { // if serve has set up the tree, it must match what we // expect if (tree_id != serve_result->tree) { (*logger)(fmt::format("Mismatch in served root tree " "id:\nexpected {}, but got {}", tree_id, serve_result->tree), /*fatal=*/true); return; } // we only need the serve endpoint to try to set up the // root, as we will check the remote CAS for the resulting // tree anyway } else { // check if serve failure was due to distdir content not // being found or it is otherwise fatal if (serve_result.error() == GitLookupError::Fatal) { (*logger)( fmt::format("Serve endpoint failed to set up root " "from known distdir content {}", key.content_id), /*fatal=*/true); return; } } } // we could not set the root as present using the CAS tree // invariant, so now we need to ensure we have all individual blobs content_cas_map->ConsumeAfterKeysReady( ts, *key.repos_to_fetch, [distdir_tree_id_file, key, import_to_git_map, native_storage_config, native_storage, ts, setter, logger]([[maybe_unused]] auto const& values) { // archive blobs are in CAS ImportFromCASAndSetRoot(key, *native_storage_config, *native_storage, distdir_tree_id_file, import_to_git_map, ts, setter, logger); }, [logger, content_id = key.content_id](auto const& msg, bool fatal) { (*logger)(fmt::format("While fetching archives for distdir " "content {}:\n{}", content_id, msg), fatal); }); } }; return AsyncMapConsumer>( distdir_to_git, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/distdir_git_map.hpp000066400000000000000000000061301516554100600306250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" struct DistdirInfo { std::string content_id; /* key */ std::shared_ptr> content_list; std::shared_ptr> repos_to_fetch; // name of repository for which work is done; used in progress reporting std::string origin; // create an absent root bool absent{}; /* key */ [[nodiscard]] auto operator==(const DistdirInfo& other) const noexcept -> bool { return content_id == other.content_id and absent == other.absent; } }; /// \brief Maps a list of repositories belonging to a distdir to its /// corresponding workspace root and indication whether this was a cache hit. using DistdirGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateDistdirGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, gsl::not_null const& critical_git_op_map, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& native_storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const DistdirInfo& dd) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, dd.content_id); hash_combine(&seed, dd.absent); return seed; } }; } // namespace std #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/foreign_file_git_map.cpp000066400000000000000000000270651516554100600316200ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/foreign_file_git_map.hpp" #include #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { void WithRootImportedToGit(ForeignFileInfo const& key, StorageConfig const& storage_config, std::pair const& result, ForeignFileGitMap::SetterPtr const& setter, ForeignFileGitMap::LoggerPtr const& logger) { if (not result.second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile(storage_config, key.archive.content_hash.Hash(), key.name, key.executable); auto cache_written = StorageUtils::WriteTreeIDFile(tree_id_file, result.first); if (not cache_written) { (*logger)( fmt::format("Failed to write cache file {}", tree_id_file.string()), /*fatal=*/false); } (*setter)( std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, result.first, storage_config.GitRoot().string()}), /*is_cache_hit=*/false)); } void WithFetchedFile(ForeignFileInfo const& key, gsl::not_null const& storage_config, Storage const& storage, gsl::not_null const& import_to_git_map, gsl::not_null const& ts, ForeignFileGitMap::SetterPtr const& setter, ForeignFileGitMap::LoggerPtr const& logger) { auto tmp_dir = storage_config->CreateTypedTmpDir("foreign-file"); auto const& cas = storage.CAS(); auto digest = ArtifactDigest{key.archive.content_hash, 0}; auto content_cas_path = cas.BlobPath(digest, key.executable); if (not content_cas_path) { (*logger)( fmt::format("Failed to locally find {} after fetching for repo {}", key.archive.content_hash.Hash(), nlohmann::json(key.archive.origin).dump()), true); return; } auto did_create_hardlink = FileSystemManager::CreateFileHardlink( *content_cas_path, tmp_dir->GetPath() / key.name, LogLevel::Warning); if (not did_create_hardlink) { (*logger)(fmt::format( "Failed to hard link {} as {} in temporary directory {}", content_cas_path->string(), nlohmann::json(key.name).dump(), tmp_dir->GetPath().string()), true); return; } CommitInfo c_info{ tmp_dir->GetPath(), fmt::format("foreign file at {}", nlohmann::json(key.name).dump()), key.archive.content_hash.Hash()}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive key, storage_config, setter, logger](auto const& values) { WithRootImportedToGit( key, *storage_config, *values[0], setter, logger); }, [logger, target_path = tmp_dir->GetPath()](auto const& msg, bool fatal) { (*logger)(fmt::format("While importing target {} to Git:\n{}", target_path.string(), msg), fatal); }); } void UseCacheHit(StorageConfig const& storage_config, const std::string& tree_id, ForeignFileGitMap::SetterPtr const& setter) { // We keep the invariant, that, whenever a cache entry is written, // the root is in our git root; in particular, the latter is present, // initialized, etc; so we can directly write the result. (*setter)( std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id, storage_config.GitRoot().string()}), /*is_cache_hit=*/true)); } void HandleAbsentForeignFile(ForeignFileInfo const& key, ServeApi const* serve, ForeignFileGitMap::SetterPtr const& setter, ForeignFileGitMap::LoggerPtr const& logger) { // Compute tree in memory GitRepo::tree_entries_t entries{}; auto raw_id = FromHexString(key.archive.content_hash.Hash()); if (not raw_id) { (*logger)(fmt::format("Failure converting {} to raw id.", key.archive.content_hash.Hash()), true); return; } entries[*raw_id].emplace_back( key.name, key.executable ? ObjectType::Executable : ObjectType::File); auto tree = GitRepo::CreateShallowTree(entries); if (not tree) { (*logger)(fmt::format("Failure to construct in-memory tree with entry " "{} at place {}", key.archive.content_hash.Hash(), nlohmann::json(key.name).dump()), true); return; } auto tree_id = ToHexString(tree->first); if (serve != nullptr) { auto const has_tree = serve->CheckRootTree(tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve endpoint knows tree " "{} failed.", tree_id), /*fatal=*/true); return; } if (*has_tree) { (*setter)(std::pair( nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } auto const serve_result = serve->RetrieveTreeFromForeignFile( key.archive.content_hash.Hash(), key.name, key.executable); if (serve_result) { // if serve has set up the tree, it must match what we expect if (tree_id != serve_result->tree) { (*logger)(fmt::format("Mismatch in served root tree " "id: expected {}, but got {}", tree_id, serve_result->tree), /*fatal=*/true); return; } // set workspace root as absent (*setter)(std::pair( nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}), /*is_cache_hit=*/false)); return; } if (serve_result.error() == GitLookupError::Fatal) { (*logger)(fmt::format("Serve endpoint failed to set up root " "from known foreign-file content {}", key.archive.content_hash.Hash()), /*fatal=*/true); return; } } else { (*logger)(fmt::format("Workspace root {} marked absent but no " "serve endpoint provided.", tree_id), /*fatal=*/false); } (*setter)( std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}), false /*no cache hit*/)); } } // namespace [[nodiscard]] auto CreateForeignFileGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, ServeApi const* serve, gsl::not_null const& storage_config, gsl::not_null const& storage, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap { auto setup_foreign_file = [content_cas_map, import_to_git_map, fetch_absent, serve, storage, storage_config](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { if (key.absent and not fetch_absent) { HandleAbsentForeignFile(key, serve, setter, logger); return; } auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( *storage_config, key.archive.content_hash.Hash(), key.name, key.executable); if (FileSystemManager::Exists(tree_id_file)) { auto tree_id = FileSystemManager::ReadFile(tree_id_file); if (not tree_id) { (*logger)(fmt::format("Failed to read tree id from file {}", tree_id_file.string()), /*fatal=*/true); return; } UseCacheHit(*storage_config, *tree_id, setter); return; } content_cas_map->ConsumeAfterKeysReady( ts, {key.archive}, [key, import_to_git_map, storage, storage_config, setter, logger, ts]([[maybe_unused]] auto const& values) { WithFetchedFile(key, storage_config, *storage, import_to_git_map, ts, setter, logger); }, [logger, hash = key.archive.content_hash.Hash()](auto const& msg, bool fatal) { (*logger)(fmt::format("While ensuring content {} is in " "CAS:\n{}", hash, msg), fatal); }); }; return AsyncMapConsumer>( setup_foreign_file, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/foreign_file_git_map.hpp000066400000000000000000000035341516554100600316200ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FOREIGN_FILE_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FOREIGN_FILE_GIT_MAP_HPP #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" /// \brief Maps a foreign file to the resulting Git tree WS root, /// together with the information whether it was a cache hit. using ForeignFileGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateForeignFileGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, ServeApi const* serve, gsl::not_null const& storage_config, gsl::not_null const& storage, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FOREIGN_FILE_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/fpath_git_map.cpp000066400000000000000000000520551516554100600302670ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/fpath_git_map.hpp" #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { /// \brief Does the serve endpoint checks and sets the workspace root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. void CheckServeAndSetRoot(std::string const& tree_id, std::string const& repo_root, bool absent, ServeApi const* serve, FilePathGitMap::SetterPtr const& ws_setter, FilePathGitMap::LoggerPtr const& logger) { // if serve endpoint is given, try to ensure it has this tree available to // be able to build against it. If root is not absent, do not fail if we // don't have a suitable remote endpoint, but warn user nonetheless. if (serve != nullptr) { auto const has_tree = serve->CheckRootTree(tree_id); if (not has_tree) { (*logger)(fmt::format("Checking that the serve endpoint knows tree " "{} failed.", tree_id), /*fatal=*/true); return; } if (not *has_tree) { auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, tree_id, /*size_unknown=*/0, /*is_tree=*/true); if (not digest) { (*logger)(std::move(digest).error(), /*fatal=*/true); return; } auto uploaded = serve->UploadTree(*digest, repo_root); // only enforce root setup on the serve endpoint if root is absent if (not uploaded.has_value()) { bool const fatal = absent or not uploaded.error().IsSyncError(); (*logger)(std::move(uploaded).error().Message(), fatal); if (fatal) { return; } } } } else { if (absent) { // give warning (*logger)(fmt::format("Workspace root {} marked absent but no " "suitable serve endpoint provided.", tree_id), /*fatal=*/false); } } // set the workspace root auto root = nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}); if (not absent) { root.emplace_back(repo_root); } (*ws_setter)(std::move(root)); } void ResolveFilePathTree( std::string const& repo_root, std::string const& target_path, std::string const& tree_hash, std::optional const& pragma_special, GitCASPtr const& source_cas, GitCASPtr const& target_cas, bool absent, gsl::not_null const& critical_git_op_map, gsl::not_null const& resolve_symlinks_map, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& ts, FilePathGitMap::SetterPtr const& ws_setter, FilePathGitMap::LoggerPtr const& logger) { if (pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( *native_storage_config, tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); if (not resolved_tree_id) { (*logger)( fmt::format("Failed to read resolved tree id from file {}", tree_id_file.string()), /*fatal=*/true); return; } // if serve endpoint is given, try to ensure it has this tree // available to be able to build against it; the tree is resolved, // so it is in our Git cache CheckServeAndSetRoot(*resolved_tree_id, native_storage_config->GitRoot().string(), absent, serve, ws_setter, logger); } else { // resolve tree resolve_symlinks_map->ConsumeAfterKeysReady( ts, {GitObjectToResolve(tree_hash, ".", *pragma_special, /*known_info=*/std::nullopt, source_cas, target_cas)}, [critical_git_op_map, tree_hash, tree_id_file, absent, serve, native_storage_config, ts, ws_setter, logger](auto const& hashes) { auto const& resolved_tree_id = hashes[0]->id; // keep tree alive in Git cache via a tagged commit GitOpKey op_key = { .params = { native_storage_config ->GitRoot(), // target_path resolved_tree_id, // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [resolved_tree_id, tree_id_file, absent, serve, native_storage_config, ws_setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Keep tree failed", /*fatal=*/true); return; } // cache the resolved tree in the CAS map if (not StorageUtils::WriteTreeIDFile( tree_id_file, resolved_tree_id)) { (*logger)( fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), /*fatal=*/true); return; } // if serve endpoint is given, try to ensure it has // this tree available to be able to build against // it; the resolved tree is in the Git cache CheckServeAndSetRoot( resolved_tree_id, native_storage_config->GitRoot().string(), absent, serve, ws_setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "KEEP_TREE for target {}:\n{}", target_path.string(), msg), fatal); }); }, [logger, target_path](auto const& msg, bool fatal) { (*logger)(fmt::format( "While resolving symlinks for target {}:\n{}", target_path, msg), fatal); }); } } else { // tree needs no further processing; // if serve endpoint is given, try to ensure it has this tree available // to be able to build against it CheckServeAndSetRoot( tree_hash, repo_root, absent, serve, ws_setter, logger); } } } // namespace auto CreateFilePathGitMap( std::optional const& current_subcmd, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, gsl::not_null const& resolve_symlinks_map, ServeApi const* serve, gsl::not_null const& native_storage_config, std::size_t jobs, std::string const& multi_repo_tool_name, std::string const& build_tool_name) -> FilePathGitMap { auto dir_to_git = [current_subcmd, critical_git_op_map, import_to_git_map, resolve_symlinks_map, serve, native_storage_config, multi_repo_tool_name, build_tool_name](auto ts, auto setter, auto logger, auto /*unused*/, auto const& key) { // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting repo root from path:\n{}", msg), fatal); }); // check if path is a part of a git repo auto repo_root = GitRepoRemote::GetRepoRootFromPath( key.fpath, wrapped_logger); // static function if (not repo_root) { return; } if (not repo_root->empty()) { // if repo root found // get head commit GitOpKey op_key = {.params = { *repo_root, // target_path "", // git_hash }, .op_type = GitOpType::GET_HEAD_ID}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [fpath = key.fpath, pragma_special = key.pragma_special, absent = key.absent, repo_root = std::move(*repo_root), critical_git_op_map, resolve_symlinks_map, serve, native_storage_config, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Get Git head id failed", /*fatal=*/true); return; } auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open repository {}", repo_root.string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While getting subtree from " "path:\n{}", msg), fatal); }); // get tree id auto tree_hash = git_repo->GetSubtreeFromPath( fpath, *op_result.result, wrapped_logger); if (not tree_hash) { return; } // resolve tree and set workspace root; tree gets resolved // from source repo into the Git cache, which we first need // to ensure is initialized GitOpKey op_key = {.params = { native_storage_config ->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [repo_root, fpath, tree_hash, pragma_special, source_cas = op_result.git_cas, absent, critical_git_op_map, resolve_symlinks_map, serve, native_storage_config, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git init failed", /*fatal=*/true); return; } ResolveFilePathTree( repo_root.string(), fpath.string(), *tree_hash, pragma_special, source_cas, op_result.git_cas, /*just_git_cas*/ absent, critical_git_op_map, resolve_symlinks_map, serve, native_storage_config, ts, setter, logger); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), msg), fatal); }); }, [logger, target_path = *repo_root](auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "GET_HEAD_ID for target {}:\n{}", target_path.string(), msg), fatal); }); } else { // warn if import to git is inefficient if (current_subcmd) { (*logger)(fmt::format("Inefficient Git import of file " "path \'{}\'.\nPlease consider using " "\'{} setup\' and \'{} {}\' " "separately to cache the output.", key.fpath.string(), multi_repo_tool_name, build_tool_name, *current_subcmd), /*fatal=*/false); } // it's not a git repo, so import it to git cache auto tmp_dir = native_storage_config->CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); return; } // copy folder content to tmp dir if (not FileSystemManager::CopyDirectoryImpl(key.fpath, tmp_dir->GetPath())) { (*logger)( fmt::format("Failed to copy content from directory {}", key.fpath.string()), /*fatal=*/true); return; } // do import to git CommitInfo c_info{tmp_dir->GetPath(), "file", key.fpath}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, // tmp_dir passed, to ensure folder is not removed until import // to git is done [tmp_dir, fpath = key.fpath, pragma_special = key.pragma_special, absent = key.absent, critical_git_op_map, resolve_symlinks_map, serve, native_storage_config, ts, setter, logger](auto const& values) { // check for errors if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // we only need the tree std::string tree = values[0]->first; // resolve tree and set workspace root; // we work on the Git CAS directly ResolveFilePathTree( native_storage_config->GitRoot().string(), fpath.string(), tree, pragma_special, values[0]->second, /*source_cas*/ values[0]->second, /*target_cas*/ absent, critical_git_op_map, resolve_symlinks_map, serve, native_storage_config, ts, setter, logger); }, [logger, target_path = key.fpath](auto const& msg, bool fatal) { (*logger)( fmt::format("While importing target {} to git:\n{}", target_path.string(), msg), fatal); }); } }; return AsyncMapConsumer(dir_to_git, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/fpath_git_map.hpp000066400000000000000000000057011516554100600302700ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" struct FpathInfo { std::filesystem::path fpath; /* key */ // create root based on "special" pragma value std::optional pragma_special{std::nullopt}; /* key */ // create an absent root bool absent{}; /* key */ [[nodiscard]] auto operator==(const FpathInfo& other) const noexcept -> bool { return fpath == other.fpath and pragma_special == other.pragma_special and absent == other.absent; } }; /// \brief Maps the path to a repo on the file system to its Git tree WS root. using FilePathGitMap = AsyncMapConsumer; [[nodiscard]] auto CreateFilePathGitMap( std::optional const& current_subcmd, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, gsl::not_null const& resolve_symlinks_map, ServeApi const* serve, gsl::not_null const& native_storage_config, std::size_t jobs, std::string const& multi_repo_tool_name, std::string const& build_tool_name) -> FilePathGitMap; namespace std { template <> struct hash { [[nodiscard]] auto operator()(FpathInfo const& ct) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ct.fpath); hash_combine>(&seed, ct.pragma_special); hash_combine(&seed, ct.absent); return seed; } }; } // namespace std #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/tree_id_git_map.cpp000066400000000000000000000411131516554100600305710ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/root_maps/tree_id_git_map.hpp" #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. void UploadToServeAndSetRoot( ServeApi const& serve, gsl::not_null const& native_storage_config, ArtifactDigest const& digest, bool ignore_special, TreeIdGitMap::SetterPtr const& setter, TreeIdGitMap::LoggerPtr const& logger) { auto uploaded = serve.UploadTree(digest, native_storage_config->GitRoot()); if (not uploaded.has_value()) { (*logger)(uploaded.error().Message(), /*fatal=*/true); return; } // set workspace root as absent auto root = nlohmann::json::array( {ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, digest.hash()}); (*setter)(std::pair(std::move(root), /*is_cache_hit=*/false)); } /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. void MoveCASTreeToGitAndProcess( ServeApi const& serve, gsl::not_null const& native_storage_config, ArtifactDigest const& digest, gsl::not_null const& import_to_git_map, gsl::not_null const& local_api, bool ignore_special, gsl::not_null const& ts, TreeIdGitMap::SetterPtr const& setter, TreeIdGitMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage auto tmp_dir = native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", digest.hash()), true); return; } if (not local_api->RetrieveToPaths( {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { (*logger)(fmt::format("Failed to copy git-tree {} to {}", digest.hash(), tmp_dir->GetPath().string()), true); return; } CommitInfo c_info{tmp_dir->GetPath(), "tree", digest.hash()}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [&serve, native_storage_config, tmp_dir, // keep tmp_dir alive digest, ignore_special, setter, logger](auto const& values) { if (not values[0]->second) { (*logger)("Importing to git failed", /*fatal=*/true); return; } // upload tree from Git cache to remote CAS and tell serve to set up // the root from the remote CAS tree; set root as absent on success UploadToServeAndSetRoot(serve, native_storage_config, digest, ignore_special, setter, logger); }, [logger, tmp_dir, tree_id = digest.hash()](auto const& msg, bool fatal) { (*logger)(fmt::format( "While moving git-tree {} from {} to local git:\n{}", tree_id, tmp_dir->GetPath().string(), msg), fatal); }); } } // namespace auto CreateTreeIdGitMap( gsl::not_null const& git_tree_fetch_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap { auto tree_to_git = [git_tree_fetch_map, critical_git_op_map, import_to_git_map, fetch_absent, serve, native_storage_config, local_api, remote_api](auto ts, auto setter, auto logger, auto /*unused*/, auto const& key) { // if root is actually absent, check if serve endpoint knows the tree // for building against it and only set the workspace root if tree is // found on the serve endpoint or it can be made available to it; // otherwise, error out if (key.absent and not fetch_absent) { if (serve != nullptr) { // check serve endpoint auto const has_tree = serve->CheckRootTree(key.tree_info.tree_hash.Hash()); if (not has_tree) { (*logger)(fmt::format( "Checking that the serve endpoint knows tree " "{} failed.", key.tree_info.tree_hash.Hash()), /*fatal=*/true); return; } if (*has_tree) { // set workspace root as absent auto root = nlohmann::json::array( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, key.tree_info.tree_hash.Hash()}); (*setter)( std::pair(std::move(root), /*is_cache_hit=*/false)); return; } // we cannot continue without a suitable remote set up if (remote_api == nullptr) { (*logger)( fmt::format("Cannot create workspace root {} as absent " "for the provided serve endpoint.", key.tree_info.tree_hash.Hash()), /*fatal=*/true); return; } // check if tree in already in remote CAS auto const digest = ArtifactDigest{key.tree_info.tree_hash, 0}; if (remote_api->IsAvailable(digest)) { // tell serve to set up the root from the remote CAS tree; if (serve->GetTreeFromRemote(digest)) { // set workspace root as absent auto root = nlohmann::json::array( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, key.tree_info.tree_hash.Hash()}); (*setter)( std::pair(std::move(root), /*is_cache_hit=*/false)); return; } (*logger)( fmt::format("Serve endpoint failed to create workspace " "root {} that locally was marked absent.", key.tree_info.tree_hash.Hash()), /*fatal=*/true); return; } // check if tree is in Git cache; // ensure Git cache exists GitOpKey op_key = { .params = { native_storage_config->GitRoot(), // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [serve, native_storage_config, digest, import_to_git_map, local_api, key, ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { (*logger)("Git cache init failed", /*fatal=*/true); return; } // Open fake tmp repo to check if tree is known to Git // cache auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { (*logger)( fmt::format( "Could not open repository {}", native_storage_config->GitRoot().string()), /*fatal=*/true); return; } // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { (*logger)( fmt::format("While checking tree " "exists in Git cache:\n{}", msg), fatal); }); // check if the desired tree ID is in Git cache auto tree_found = git_repo->CheckTreeExists( key.tree_info.tree_hash.Hash(), wrapped_logger); if (not tree_found) { // errors encountered return; } if (*tree_found) { // upload tree from Git cache to remote CAS and tell // serve to set up the root from the remote CAS // tree, then set root as absent UploadToServeAndSetRoot(*serve, native_storage_config, digest, key.ignore_special, setter, logger); // done! return; } // check if tree is known to local CAS if (local_api->IsAvailable(digest)) { // Move tree locally from CAS to Git cache, then // continue processing it by UploadToServeAndSetRoot MoveCASTreeToGitAndProcess(*serve, native_storage_config, digest, import_to_git_map, local_api, key.ignore_special, ts, setter, logger); // done! return; } // tree is not know locally, so we cannot // provide it to the serve endpoint and thus we // cannot create the absent root (*logger)(fmt::format("Cannot create workspace root " "{} as absent for the provided " "serve endpoint.", key.tree_info.tree_hash.Hash()), /*fatal=*/true); }, [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " "ENSURE_INIT bare for target {}:\n{}", target_path.string(), msg), fatal); }); // done! return; } // give warning that serve endpoint is missing (*logger)(fmt::format("Workspace root {} marked absent but no " "suitable serve endpoint provided.", key.tree_info.tree_hash.Hash()), /*fatal=*/false); // set workspace root as absent auto root = nlohmann::json::array( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, key.tree_info.tree_hash.Hash()}); (*setter)(std::pair(std::move(root), false)); return; } // if root is not absent, proceed with usual fetch logic: check locally, // check serve endpoint, check remote-execution endpoint, and lastly // default to network git_tree_fetch_map->ConsumeAfterKeysReady( ts, {key.tree_info}, [native_storage_config, key, setter](auto const& values) { // tree is now in Git cache; // get cache hit info auto is_cache_hit = *values[0]; // set the workspace root as present (*setter)( std::pair(nlohmann::json::array( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, key.tree_info.tree_hash.Hash(), native_storage_config->GitRoot().string()}), is_cache_hit)); }, [logger, hash = key.tree_info.tree_hash.Hash()](auto const& msg, bool fatal) { (*logger)(fmt::format( "While ensuring git-tree {} is in Git cache:\n{}", hash, msg), fatal); }); }; return AsyncMapConsumer>( tree_to_git, jobs); } just-buildsystem-justbuild-b1fb5fa/src/other_tools/root_maps/tree_id_git_map.hpp000066400000000000000000000055631516554100600306070ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" struct TreeIdInfo { GitTreeInfo tree_info; /* key */ // create root that ignores symlinks bool ignore_special{}; /* key */ // create an absent root bool absent{}; /* key */ [[nodiscard]] auto operator==(const TreeIdInfo& other) const -> bool { return tree_info == other.tree_info and ignore_special == other.ignore_special and absent == other.absent; } }; namespace std { template <> struct hash { [[nodiscard]] auto operator()(const TreeIdInfo& ti) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, ti.tree_info); hash_combine(&seed, ti.ignore_special); hash_combine(&seed, ti.absent); return seed; } }; } // namespace std /// \brief Maps a known tree provided through a generic command to its /// workspace root and the information whether it was a cache hit. using TreeIdGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateTreeIdGitMap( gsl::not_null const& git_tree_fetch_map, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, bool fetch_absent, ServeApi const* serve, gsl::not_null const& native_storage_config, gsl::not_null const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/000077500000000000000000000000001516554100600241075ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/TARGETS000066400000000000000000000071621516554100600251510ustar00rootroot00000000000000{ "curl_context": { "type": ["@", "rules", "CC", "library"] , "name": ["curl_context"] , "hdrs": ["curl_context.hpp"] , "srcs": ["curl_context.cpp"] , "stage": ["src", "other_tools", "utils"] , "private-deps": [ ["", "libcurl"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "curl_easy_handle": { "type": ["@", "rules", "CC", "library"] , "name": ["curl_easy_handle"] , "hdrs": ["curl_easy_handle.hpp"] , "srcs": ["curl_easy_handle.cpp"] , "deps": [ "curl_context" , ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "log_level"] ] , "stage": ["src", "other_tools", "utils"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["", "libcurl"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "logging"] ] } , "curl_url_handle": { "type": ["@", "rules", "CC", "library"] , "name": ["curl_url_handle"] , "hdrs": ["curl_url_handle.hpp"] , "srcs": ["curl_url_handle.cpp"] , "deps": ["curl_context", ["@", "gsl", "", "gsl"]] , "stage": ["src", "other_tools", "utils"] , "private-deps": [ ["", "libcurl"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "content": { "type": ["@", "rules", "CC", "library"] , "name": ["content"] , "hdrs": ["content.hpp"] , "deps": [ "curl_easy_handle" , ["@", "fmt", "", "fmt"] , ["src/buildtool/common", "user_structs"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/logging", "log_level"] , ["src/other_tools/just_mr", "mirrors"] , ["src/utils/cpp", "expected"] ] , "stage": ["src", "other_tools", "utils"] } , "parse_archive": { "type": ["@", "rules", "CC", "library"] , "name": ["parse_archive"] , "hdrs": ["parse_archive.hpp"] , "srcs": ["parse_archive.cpp"] , "deps": [ ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] , ["src/buildtool/file_system/symlinks", "pragma_special"] , ["src/utils/cpp", "path"] ] , "stage": ["src", "other_tools", "utils"] } , "parse_git_tree": { "type": ["@", "rules", "CC", "library"] , "name": ["parse_git_tree"] , "hdrs": ["parse_git_tree.hpp"] , "srcs": ["parse_git_tree.cpp"] , "deps": [ ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/crypto", "hash_info"] ] , "stage": ["src", "other_tools", "utils"] } , "parse_precomputed_root": { "type": ["@", "rules", "CC", "library"] , "name": ["parse_precomputed_root"] , "hdrs": ["parse_precomputed_root.hpp"] , "srcs": ["parse_precomputed_root.cpp"] , "deps": [ ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/file_system", "precomputed_root"] , ["src/utils/cpp", "expected"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] ] , "stage": ["src", "other_tools", "utils"] } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/content.hpp000066400000000000000000000071661516554100600263040ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CONTENT_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_CONTENT_HPP #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/utils/curl_easy_handle.hpp" #include "src/utils/cpp/expected.hpp" // Utilities related to the content of an archive /// \brief Fetches a file from the internet and stores its content in memory. /// \returns the content. [[nodiscard]] static inline auto NetworkFetch(std::string const& fetch_url, CAInfoPtr const& ca_info) noexcept -> std::optional { auto curl_handle = CurlEasyHandle::Create( ca_info->no_ssl_verify, ca_info->ca_bundle, LogLevel::Debug); if (not curl_handle) { return std::nullopt; } return curl_handle->DownloadToString(fetch_url); } /// \brief Fetches a file from the internet and stores its content in memory. /// Tries not only a given remote, but also all associated remote locations. /// \returns The fetched data on success or an unexpected error as string. [[nodiscard]] static inline auto NetworkFetchWithMirrors( std::string const& fetch_url, std::vector const& mirrors, CAInfoPtr const& ca_info, MirrorsPtr const& additional_mirrors) noexcept -> expected { // keep all remotes tried, to report in case fetch fails std::string remotes_buffer{}; std::optional data{std::nullopt}; // try repo url auto all_mirrors = std::vector({fetch_url}); // try repo mirrors afterwards all_mirrors.insert(all_mirrors.end(), mirrors.begin(), mirrors.end()); if (auto preferred_hostnames = MirrorsUtils::GetPreferredHostnames(additional_mirrors); not preferred_hostnames.empty()) { all_mirrors = MirrorsUtils::SortByHostname(all_mirrors, preferred_hostnames); } // always try local mirrors first auto local_mirrors = MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_url); all_mirrors.insert( all_mirrors.begin(), local_mirrors.begin(), local_mirrors.end()); for (auto const& mirror : all_mirrors) { if (data = NetworkFetch(mirror, ca_info); data) { break; } // add local mirror to buffer remotes_buffer.append(fmt::format("\n> {}", mirror)); } if (not data) { return unexpected{remotes_buffer}; } return *data; } template [[nodiscard]] static inline auto GetContentHash( std::string const& data) noexcept -> std::string { auto hasher = Hasher::Create(kType); hasher->Update(data); auto digest = std::move(*hasher).Finalize(); return digest.HexString(); } #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CONTENT_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_context.cpp000066400000000000000000000021301516554100600273200ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/curl_context.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" extern "C" { #include } CurlContext::CurlContext() noexcept : initialized_{curl_global_init(CURL_GLOBAL_DEFAULT) >= 0} { if (not initialized_) { Logger::Log(LogLevel::Error, "initializing libcurl failed"); } } CurlContext::~CurlContext() noexcept { if (initialized_) { curl_global_cleanup(); } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_context.hpp000066400000000000000000000024571516554100600273410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP /// \brief Maintainer of a libcurl state. /// Classes, static methods, and global functions dealing with curl operations /// should create a CurlContext before using libcurl. class CurlContext { public: // prohibit moves and copies CurlContext(CurlContext const&) = delete; CurlContext(CurlContext&& other) = delete; auto operator=(CurlContext const&) = delete; auto operator=(CurlContext&& other) = delete; CurlContext() noexcept; ~CurlContext() noexcept; private: bool initialized_{false}; }; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_easy_handle.cpp000066400000000000000000000240241516554100600301160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/curl_easy_handle.hpp" #include #include #include #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" extern "C" { #include "curl/curl.h" } void curl_easy_closer(gsl::owner curl) { curl_easy_cleanup(curl); } namespace { auto read_stream_data(gsl::not_null const& stream) noexcept -> std::string { // obtain stream size std::fseek(stream, 0, SEEK_END); auto size = std::ftell(stream); auto pos = std::fseek(stream, 0, SEEK_SET); if (pos != 0) { Logger::Log(LogLevel::Warning, "Rewinding temporary file for curl log failed."); } // create string buffer to hold stream content std::string content(static_cast(size), '\0'); // read stream content into string buffer auto n = std::fread(content.data(), 1, content.size(), stream); if (n != static_cast(size)) { Logger::Log(LogLevel::Warning, "Reading curl log from temporary file failed: read only {} " "bytes while {} were expected", n, size); } return content; } } // namespace auto CurlEasyHandle::Create(LogLevel log_level) noexcept -> std::shared_ptr { return Create(false, std::nullopt, log_level); } auto CurlEasyHandle::Create( bool no_ssl_verify, std::optional const& ca_bundle, LogLevel log_level) noexcept -> std::shared_ptr { try { auto curl = std::make_shared(); auto* handle = curl_easy_init(); if (handle == nullptr) { return nullptr; } curl->handle_.reset(handle); // store CA info curl->no_ssl_verify_ = no_ssl_verify; curl->ca_bundle_ = ca_bundle; // store log level curl->log_level_ = log_level; return curl; } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "create curl easy handle failed with:\n{}", ex.what()); return nullptr; } } auto CurlEasyHandle::EasyWriteToFile(gsl::owner data, std::size_t size, std::size_t nmemb, gsl::owner userptr) -> std::streamsize { auto actual_size = static_cast(size * nmemb); auto* file = static_cast(userptr); file->write(data, actual_size); // append chunk return actual_size; } auto CurlEasyHandle::EasyWriteToString(gsl::owner data, std::size_t size, std::size_t nmemb, gsl::owner userptr) -> std::streamsize { size_t actual_size = size * nmemb; (static_cast(userptr))->append(data, actual_size); return static_cast(actual_size); } auto CurlEasyHandle::DownloadToFile( std::string const& url, std::filesystem::path const& file_path) noexcept -> int { // create temporary file to capture curl debug output gsl::owner tmp_file = std::tmpfile(); try { // set URL // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); // ensure redirects are allowed, otherwise it might simply read empty // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); // ensure failure on error codes that otherwise might return OK // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_FAILONERROR, 1); // set callback for writing to file std::ofstream file(file_path.c_str(), std::ios::binary); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToFile); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt( handle_.get(), CURLOPT_WRITEDATA, static_cast(&file)); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_STDERR, tmp_file); // set SSL options // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_SSL_VERIFYPEER, static_cast(not no_ssl_verify_)); if (ca_bundle_) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt( handle_.get(), CURLOPT_CAINFO, ca_bundle_->c_str()); } // perform download auto res = curl_easy_perform(handle_.get()); // close file file.close(); // check result if (res != CURLE_OK) { // cleanup failed downloaded file, if created [[maybe_unused]] auto tmp_res = FileSystemManager::RemoveFile(file_path); Logger::Log(log_level_, [&tmp_file]() { return fmt::format("curl download to file failed:\n{}", read_stream_data(tmp_file)); }); std::fclose(tmp_file); return 1; } // print curl debug output if log level is tracing Logger::Log(LogLevel::Trace, [&tmp_file]() { return fmt::format("stderr of curl downloading to file:\n{}", read_stream_data(tmp_file)); }); std::fclose(tmp_file); return res; } catch (std::exception const& ex) { Logger::Log(log_level_, [&ex, &tmp_file]() { return fmt::format( "curl download to file failed with:\n{}\n" "while performing:\n{}", ex.what(), read_stream_data(tmp_file)); }); std::fclose(tmp_file); return 1; } } auto CurlEasyHandle::DownloadToString(std::string const& url) noexcept -> std::optional { // create temporary file to capture curl debug output gsl::owner tmp_file = std::tmpfile(); try { // set URL // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); // ensure redirects are allowed, otherwise it might simply read empty // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); // ensure failure on error codes that otherwise might return OK // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_FAILONERROR, 1); // set callback for writing to string std::string content{}; // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt( handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToString); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt( handle_.get(), CURLOPT_WRITEDATA, static_cast(&content)); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_STDERR, tmp_file); // set SSL options // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt(handle_.get(), CURLOPT_SSL_VERIFYPEER, static_cast(not no_ssl_verify_)); if (ca_bundle_) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) curl_easy_setopt( handle_.get(), CURLOPT_CAINFO, ca_bundle_->c_str()); } // perform download auto res = curl_easy_perform(handle_.get()); // check result if (res != CURLE_OK) { Logger::Log(log_level_, [&tmp_file]() { return fmt::format("curl download to string failed:\n{}", read_stream_data(tmp_file)); }); std::fclose(tmp_file); return std::nullopt; } // print curl debug output if log level is tracing Logger::Log(LogLevel::Trace, [&tmp_file]() { return fmt::format("stderr of curl downloading to string:\n{}", read_stream_data(tmp_file)); }); std::fclose(tmp_file); return content; } catch (std::exception const& ex) { Logger::Log(log_level_, [&ex, &tmp_file]() { return fmt::format( "curl download to string failed with:\n{}\n" "while performing:\n{}", ex.what(), read_stream_data(tmp_file)); }); std::fclose(tmp_file); return std::nullopt; } } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_easy_handle.hpp000066400000000000000000000073361516554100600301320ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/other_tools/utils/curl_context.hpp" extern "C" { #if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) using CURL = struct Curl_easy; #else using CURL = void; #endif } void curl_easy_closer(gsl::owner curl); class CurlEasyHandle { public: CurlEasyHandle() noexcept = default; ~CurlEasyHandle() noexcept = default; // prohibit moves and copies CurlEasyHandle(CurlEasyHandle const&) = delete; CurlEasyHandle(CurlEasyHandle&& other) = delete; auto operator=(CurlEasyHandle const&) = delete; auto operator=(CurlEasyHandle&& other) = delete; /// \brief Create a CurlEasyHandle object [[nodiscard]] auto static Create( LogLevel log_level = LogLevel::Error) noexcept -> std::shared_ptr; /// \brief Create a CurlEasyHandle object with non-default CA info [[nodiscard]] auto static Create( bool no_ssl_verify, std::optional const& ca_bundle, LogLevel log_level = LogLevel::Error) noexcept -> std::shared_ptr; /// \brief Download file from URL into given file_path. /// Will perform cleanup (i.e., remove empty file) in case download fails. /// Returns 0 if successful. [[nodiscard]] auto DownloadToFile( std::string const& url, std::filesystem::path const& file_path) noexcept -> int; /// \brief Download file from URL into string as binary. /// Returns the content or nullopt if download failure. [[nodiscard]] auto DownloadToString(std::string const& url) noexcept -> std::optional; private: // IMPORTANT: the CurlContext must to be initialized before any curl object! CurlContext curl_context_; std::unique_ptr handle_{ nullptr, curl_easy_closer}; // allow also non-fatal logging of curl operations LogLevel log_level_{}; bool no_ssl_verify_{false}; std::optional ca_bundle_{std::nullopt}; /// \brief Overwrites write_callback to redirect to file instead of stdout. [[nodiscard]] auto static EasyWriteToFile(gsl::owner data, std::size_t size, std::size_t nmemb, gsl::owner userptr) -> std::streamsize; /// \brief Overwrites write_callback to redirect to string instead of /// stdout. [[nodiscard]] auto static EasyWriteToString(gsl::owner data, std::size_t size, std::size_t nmemb, gsl::owner userptr) -> std::streamsize; }; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_url_handle.cpp000066400000000000000000000776511516554100600277750ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/curl_url_handle.hpp" #include #include #include #include #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" extern "C" { #include "curl/curl.h" } void curl_url_closer(gsl::owner handle) { curl_url_cleanup(handle); } namespace { /// \brief Compares the two hosts as '.'-delimited substrings until there is a /// mismatch. Wildcard ("*") matches any substring. /// Returns a success flag. [[nodiscard]] auto HostsMatch(std::string const& key_host, std::string const& url_host) noexcept -> bool { // split key host std::vector key_tokens{}; std::string token{}; std::istringstream iss(key_host); while (std::getline(iss, token, '.')) { key_tokens.emplace_back(token); } // split url host std::vector url_tokens{}; iss = std::istringstream{url_host}; while (std::getline(iss, token, '.')) { url_tokens.emplace_back(token); } // number of tokens must match if (key_tokens.size() != url_tokens.size()) { return false; } // check for substring mismatch auto key_it = key_tokens.begin(); auto url_it = url_tokens.begin(); for (; key_it != key_tokens.end(); ++key_it, ++url_it) { if (*key_it != *url_it and *key_it != "*") { return false; } } return true; } /// \brief Compares the two paths as '/'-delimited substrings until there is a /// mismatch or the end of the key path. /// Returns the size of the key path if match successful, otherwise nullopt. [[nodiscard]] auto PathMatchSize(std::string const& key_path, std::string const& url_path) noexcept -> std::optional { // split key path std::vector key_tokens{}; std::string token{}; std::istringstream iss(key_path); while (std::getline(iss, token, '/')) { key_tokens.emplace_back(token); } // split url path std::vector url_tokens{}; iss = std::istringstream{url_path}; while (std::getline(iss, token, '/')) { url_tokens.emplace_back(token); } // key path should not have more tokens than the url path if (key_tokens.size() > url_tokens.size()) { return std::nullopt; } // check for substring mismatch auto key_it = key_tokens.begin(); auto url_it = url_tokens.begin(); for (; key_it != key_tokens.end(); ++key_it, ++url_it) { if (*key_it != *url_it) { return std::nullopt; } } // on success, return size of key path return key_path.size(); } /// \brief Parses the given string according to the scheme: /// [[.].][:] /// The parsing ignores a single leading '.' character, if present. /// Does not perform any other validity check (e.g., for port value). [[nodiscard]] auto ParseNoproxyPattern(std::string const& pattern) noexcept -> NoproxyPattern { // get the host part std::string host{}; std::istringstream iss(pattern); std::getline(iss, host, ':'); // stop at port part or end of string // check if port part exists std::optional port{std::nullopt}; if (host.size() != pattern.size()) { port = std::string( pattern.begin() + static_cast(host.size()) + 1, pattern.end()); } // remove one leading '.' char from host part, if present if (host[0] == '.') { host = std::string(host.begin() + 1, host.end()); } // split the host part std::vector host_tokens{}; std::string token{}; iss = std::istringstream(host); while (std::getline(iss, token, '.')) { host_tokens.emplace_back(token); } return NoproxyPattern{host_tokens, port}; } /// \brief Check whether a given test pattern matches a target pattern with /// respect to the matching rules for the no_proxy envariable. [[nodiscard]] auto NoproxyPatternMatches(NoproxyPattern const& test_pattern, NoproxyPattern const& target_pattern) -> bool { // check if port matches, if given if (test_pattern.port and test_pattern.port != target_pattern.port) { return false; } // host tokens must exist if (test_pattern.host_tokens.empty() or target_pattern.host_tokens.empty() or test_pattern.host_tokens.size() > target_pattern.host_tokens.size()) { return false; } // check if the host/domain substrings match, in reverse order auto test_it = test_pattern.host_tokens.end() - 1; auto target_it = target_pattern.host_tokens.end() - 1; for (; test_it != test_pattern.host_tokens.begin() - 1; --test_it, --target_it) { if (*test_it != *target_it) { return false; } } return true; } } // namespace auto CurlURLHandle::Create(std::string const& url) noexcept -> std::optional { try { auto url_h = std::make_shared(); auto* handle = curl_url(); // try to parse the given url auto rc = curl_url_set(handle, CURLUPART_URL, url.c_str(), 0U); if (rc != CURLUE_OK) { Logger::Log(LogLevel::Debug, "CurlURLHandle: parsing URL {} failed with:\n{}", url, curl_url_strerror(rc)); curl_url_cleanup(handle); return nullptr; } url_h->handle_.reset(handle); return std::make_optional(url_h); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "CurlURLHandle: creating curl URL handle failed " "unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::CreatePermissive(std::string const& url, bool use_guess_scheme, bool use_default_scheme, bool use_non_support_scheme, bool use_no_authority, bool use_path_as_is, bool use_allow_space, bool ignore_fatal) noexcept -> std::optional { try { auto url_h = std::make_shared(); auto* handle = curl_url(); // set up flags // NOLINTNEXTLINE(hicpp-signed-bitwise) auto flags{use_guess_scheme ? CURLU_GUESS_SCHEME : 0U}; if (use_default_scheme) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_DEFAULT_SCHEME; } if (use_non_support_scheme) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_NON_SUPPORT_SCHEME; } if (use_no_authority) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_NO_AUTHORITY; } if (use_path_as_is) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_PATH_AS_IS; } if (use_allow_space) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_ALLOW_SPACE; } // try to parse the given url // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) auto rc = curl_url_set(handle, CURLUPART_URL, url.c_str(), flags); if (rc != CURLUE_OK) { Logger::Log( LogLevel::Debug, "CurlURLHandle: parsing URL permissively failed with:\n{}", curl_url_strerror(rc)); curl_url_cleanup(handle); return nullptr; } url_h->handle_.reset(handle); return std::make_optional(url_h); } catch (std::exception const& ex) { Logger::Log(ignore_fatal ? LogLevel::Debug : LogLevel::Error, "CurlURLHandle: creating permissive curl URL handle failed " "unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::Duplicate() noexcept -> CurlURLHandlePtr { try { auto url_h = std::make_shared(); url_h->handle_.reset(curl_url_dup(handle_.get())); return url_h; } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "CurlURLHandle: duplicating curl URL handle failed " "unexpectedly with:\n{}", ex.what()); return nullptr; } } auto CurlURLHandle::GetURL(bool use_default_port, bool use_default_scheme, bool use_no_default_port, bool ignore_fatal) noexcept -> std::optional { try { // set up flags // NOLINTNEXTLINE(hicpp-signed-bitwise) auto flags{use_default_port ? CURLU_DEFAULT_PORT : 0U}; if (use_default_scheme) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_DEFAULT_SCHEME; } if (use_no_default_port) { // NOLINTNEXTLINE(hicpp-signed-bitwise) flags |= CURLU_NO_DEFAULT_PORT; } // get the URL // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url = nullptr; auto rc = curl_url_get(handle_.get(), CURLUPART_URL, &url, flags); if (rc != CURLUE_OK) { Logger::Log(ignore_fatal ? LogLevel::Debug : LogLevel::Error, "CurlURLHandle: retrieving URL failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } std::string url_str{url}; // free memory curl_free(url); return url_str; } catch (std::exception const& ex) { Logger::Log( ignore_fatal ? LogLevel::Debug : LogLevel::Error, "CurlURLHandle: retrieving URL failed unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::GetScheme(bool use_default_scheme) noexcept -> std::optional { try { // NOLINTNEXTLINE(hicpp-signed-bitwise) auto flags{use_default_scheme ? CURLU_DEFAULT_SCHEME : 0U}; // get the scheme // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* scheme = nullptr; auto rc = curl_url_get(handle_.get(), CURLUPART_SCHEME, &scheme, flags); if (rc != CURLUE_OK and rc != CURLUE_NO_SCHEME) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving scheme failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto res = OptionalString{std::nullopt}; if (rc != CURLUE_NO_SCHEME) { res = OptionalString{std::string{scheme}}; } // free memory curl_free(scheme); return res; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "CurlURLHandle: retrieving scheme failed unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::GetConfigStructFromKey(std::string const& key) noexcept -> std::optional { try { auto parsed_key = Create(key); if (not parsed_key) { return std::nullopt; // report exception } if (*parsed_key == nullptr) { return nullptr; // unparsable key } // populate all useful components // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* field = nullptr; auto* h = parsed_key.value()->handle_.get(); auto gconfig = std::make_shared(); auto rc = curl_url_get(h, CURLUPART_SCHEME, &field, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_SCHEME) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving scheme in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_SCHEME) { gconfig->scheme = std::string(field); } curl_free(field); field = nullptr; rc = curl_url_get(h, CURLUPART_USER, &field, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_USER) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving user in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_USER) { gconfig->user = std::string(field); } curl_free(field); field = nullptr; rc = curl_url_get(h, CURLUPART_HOST, &field, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_HOST) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving host in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_HOST) { gconfig->host = std::string(field); } curl_free(field); field = nullptr; rc = curl_url_get(h, CURLUPART_PORT, &field, // NOLINTNEXTLINE(hicpp-signed-bitwise) CURLU_DEFAULT_PORT); // enforce port existence if (rc != CURLUE_OK and rc != CURLUE_NO_PORT) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving port in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_PORT) { gconfig->port = std::string(field); } curl_free(field); field = nullptr; // stored path will contain also query and fragment, if existing, and // must end with a '/' rc = curl_url_get(h, CURLUPART_PATH, &field, 0U); if (rc != CURLUE_OK) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving path in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto running_path = std::filesystem::path{"/"} / std::string(field); curl_free(field); field = nullptr; rc = curl_url_get(h, CURLUPART_QUERY, &field, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_QUERY) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving query in get config struct " "failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_QUERY) { running_path += std::string("?") + std::string(field); } curl_free(field); field = nullptr; rc = curl_url_get(h, CURLUPART_FRAGMENT, &field, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_FRAGMENT) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving fragment in get config " "struct failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } if (rc != CURLUE_NO_FRAGMENT) { running_path += std::string("#") + std::string(field); } curl_free(field); running_path /= ""; // make sure it ends with a '/' gconfig->path = running_path; return gconfig; } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "CurlURLHandle: get config struct from parsed key failed " "unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::ParseConfigKey(std::string const& key) noexcept -> std::optional { try { // if key has no asterisks, parse as usual if (key.find('*') == std::string::npos) { return GetConfigStructFromKey(key); } // replace all '*' wildcards with '.' std::string tmp_key{key}; std::replace(tmp_key.begin(), tmp_key.end(), '*', '.'); // parse and extract hostname auto tmp_parsed = Create(tmp_key); if (not tmp_parsed) { return std::nullopt; // exception } if (tmp_parsed == nullptr) { return nullptr; // unparsable } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* host_ptr = nullptr; auto rc = curl_url_get( tmp_parsed.value()->handle_.get(), CURLUPART_HOST, &host_ptr, 0U); if (rc != CURLUE_OK) { Logger::Log( LogLevel::Error, "CurlURLHandle: retrieving host in parse config key failed " "with:\n{}", curl_url_strerror(rc)); return std::nullopt; } std::string parsed_host{host_ptr}; curl_free(host_ptr); // release memory // create regex to find all possible matches of the parsed host in the // original key, where any '.' can also be a '*' std::stringstream pattern{}; std::size_t old_index{}; std::size_t index{}; while ((index = parsed_host.find('.', old_index)) != std::string::npos) { pattern << parsed_host.substr(old_index, index - old_index); pattern << R"([\.\*])"; old_index = index + 1; } pattern << parsed_host.substr(old_index); std::regex re(pattern.str()); // for every match, replace the parsed host in the found position and // try to parse as usual std::size_t host_len = parsed_host.length(); for (auto it = std::sregex_iterator(key.begin(), key.end(), re); it != std::sregex_iterator(); ++it) { std::string new_key{key}; new_key.replace(static_cast(it->position()), host_len, parsed_host); // try to parse new key auto try_config_key = GetConfigStructFromKey(new_key); if (try_config_key and *try_config_key != nullptr) { // replace the parsed hostname with the match try_config_key.value()->host = it->str(); return try_config_key; } } // no match was parsable return nullptr; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "CurlURLHandle: parse config key failed unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::MatchConfigKey(std::string const& key) noexcept -> std::optional { try { std::size_t host_len{}; bool user_matched{false}; // parse the given key auto parsed_key = ParseConfigKey(key); if (not parsed_key) { return std::nullopt; // an exception occurred that shouldn't have } if (*parsed_key == nullptr) { return ConfigKeyMatchDegree{}; // non-parsable, so return no match } // check that scheme matches // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_scheme = nullptr; auto rc = curl_url_get(handle_.get(), CURLUPART_SCHEME, &url_scheme, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_SCHEME) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url scheme in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto url_scheme_str = url_scheme == nullptr ? std::nullopt : std::make_optional(url_scheme); curl_free(url_scheme); if (parsed_key.value()->scheme != url_scheme_str) { return ConfigKeyMatchDegree{}; // mismatch } // check the user, if the config key has the field if (parsed_key.value()->user) { // check the user field in stored URL // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_user = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_USER, &url_user, 0U); if (rc != CURLUE_OK) { // if key has user field, url must as well Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url user in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto url_user_str = url_user == nullptr ? std::nullopt : std::make_optional(url_user); curl_free(url_user); if (not url_user_str or parsed_key.value()->user != *url_user_str) { return ConfigKeyMatchDegree{}; // mismatch } // signal the match user_matched = true; } // check that host/domain name matches // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_host = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_HOST, &url_host, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_HOST) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url host in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto url_host_str = url_host == nullptr ? std::nullopt : std::make_optional(url_host); curl_free(url_host); if (parsed_key.value()->host != url_host_str) { if (not(parsed_key.value()->host and url_host_str and HostsMatch(parsed_key.value()->host.value(), *url_host_str))) { return ConfigKeyMatchDegree{}; // mismatch } } // store matched host length host_len += parsed_key.value()->host ? parsed_key.value()->host->size() : 0U; // check port match; get with default value if not existing // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_port = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_PORT, &url_port, // NOLINTNEXTLINE(hicpp-signed-bitwise) CURLU_DEFAULT_PORT); // enforce port existence if (rc != CURLUE_OK) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url port in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } auto url_port_str = url_port == nullptr ? std::nullopt : std::make_optional(url_port); curl_free(url_port); if (parsed_key.value()->port != url_port_str) { return ConfigKeyMatchDegree{}; // mismatch } // check path match; this is done up to any '/'-delimited prefix; // we need the complete path, so path + query + fragment (if existing) // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_path = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_PATH, &url_path, 0U); if (rc != CURLUE_OK) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url path in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } // parsed path is never empty auto url_path_str = std::filesystem::path{"/"} / std::string(url_path); curl_free(url_path); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_query = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_QUERY, &url_query, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_QUERY) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url query in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } // append to path url_path_str += url_query == nullptr ? std::string() : std::string("?") + std::string(url_query); curl_free(url_query); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_fragment = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_FRAGMENT, &url_fragment, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_FRAGMENT) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url fragment in matching " "config key failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } // append to path url_path_str += url_fragment == nullptr ? std::string() : std::string("#") + std::string(url_fragment); curl_free(url_fragment); // make sure path ends with '/' for comparison purposes url_path_str /= ""; auto path_len = PathMatchSize(parsed_key.value()->path.string(), url_path_str.string()); if (not path_len) { return ConfigKeyMatchDegree{}; // paths do not match } // key matches; success! return ConfigKeyMatchDegree{ true /*matched*/, host_len, *path_len, user_matched}; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "CurlURLHandle: match config key failed unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::NoproxyStringMatches(std::string const& no_proxy) noexcept -> std::optional { try { // split no_proxy string by both spaces and commas std::vector patterns{}; std::string token1{}; std::istringstream iss1(no_proxy); // split by spaces while (std::getline(iss1, token1, ' ')) { std::istringstream iss2(token1); std::string token2{}; // for each such token, split by commas while (std::getline(iss2, token2, ',')) { if (not token2.empty()) { patterns.emplace_back(token2); } } } // get the stored URL host (mandatory) and port (optional) as a // NoproxyPattern object // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_host = nullptr; auto rc = curl_url_get(handle_.get(), CURLUPART_HOST, &url_host, 0U); if (rc != CURLUE_OK) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url host in no_proxy string " "matching failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } std::string tmp_pattern{url_host}; curl_free(url_host); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) char* url_port = nullptr; rc = curl_url_get(handle_.get(), CURLUPART_PORT, &url_port, 0U); if (rc != CURLUE_OK and rc != CURLUE_NO_PORT) { Logger::Log(LogLevel::Error, "CurlURLHandle: retrieving url port in no_proxy string " "matching failed with:\n{}", curl_url_strerror(rc)); return std::nullopt; } // it's simpler to (re)use the existing pattern parser if (url_port != nullptr) { tmp_pattern += ":"; tmp_pattern += std::string(url_port); curl_free(url_port); } auto url_hostport_as_pattern = ParseNoproxyPattern(tmp_pattern); // check for match with any pattern for (auto const& pattern : patterns) { // ignore an empty pattern if (pattern.empty()) { continue; } // check for trivial wildcard if (pattern == "*") { return true; } // parse pattern and check for match auto parsed_pattern = ParseNoproxyPattern(pattern); if (NoproxyPatternMatches(parsed_pattern, url_hostport_as_pattern)) { return true; } } return false; } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "CurlURLHandle: no_proxy string matching failed " "unexpectedly with:\n{}", ex.what()); return std::nullopt; } } auto CurlURLHandle::GetHostname(std::string const& url) noexcept -> std::optional { try { // Allow parsing spaces in path (we only care about hostname), and do // not log error on failure. if (auto parsed_url = CreatePermissive(url, false /*use_guess_scheme*/, false /*use_default_scheme*/, false /*use_non_support_scheme*/, false /*use_no_authority*/, false /*use_path_as_is*/, true /*use_allow_space*/, true /*ignore_fatal*/)) { if (*parsed_url == nullptr) { return std::nullopt; } char* buffer{nullptr}; // NOLINT auto rc = curl_url_get( parsed_url.value()->handle_.get(), CURLUPART_HOST, &buffer, 0U); std::string hostname{}; if (buffer != nullptr) { hostname = std::string{buffer}; curl_free(buffer); } if (rc != CURLUE_OK) { Logger::Log(LogLevel::Debug, "CurlURLHandle: getting hostname from URL {} " "failed with:\n{}", url, curl_url_strerror(rc)); return std::nullopt; } return hostname; } } catch (std::exception const& ex) { Logger::Log( LogLevel::Debug, "CurlURLHandle: Getting hostname from URL failed unexpectedly " "with:\n{}", ex.what()); } return std::nullopt; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/curl_url_handle.hpp000066400000000000000000000170311516554100600277640ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_URL_HANDLE_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_URL_HANDLE_HPP #include #include #include #include #include #include #include "gsl/gsl" #include "src/other_tools/utils/curl_context.hpp" extern "C" { using CURLU = struct Curl_URL; } class CurlURLHandle; using CurlURLHandlePtr = std::shared_ptr; /// \brief Type describing a possibly missing string. Used to store a retrieved /// field of a parsed URL. using OptionalString = std::optional; void curl_url_closer(gsl::owner handle); struct GitConfigKey { OptionalString scheme{std::nullopt}; OptionalString user{std::nullopt}; // might contain wildcards OptionalString host{std::nullopt}; OptionalString port{std::nullopt}; // will include query and fragment, if existing std::filesystem::path path{"/"}; }; using GitConfigKeyPtr = std::shared_ptr; /// \brief Structure storing the information needed to quantify precedence with /// respect to the gitconfig keys matching rules. /// Non-default values are set ONLY if matching rules are satisfied. struct ConfigKeyMatchDegree { // if a matching happened; bool matched{false}; // length of config key's host field if host was matched std::size_t host_len{}; // length of config key's path field if path was matched; // comparison ends on a '/' char or the end of the path std::size_t path_len{}; // signals a match for the user field between config key and remote URL, // only if user field exists in config key bool user_matched{false}; }; /// \brief Stores the components of a valid no_proxy envariable pattern struct NoproxyPattern { // stores the substrings of the host portion of the pattern, obtained by // splitting with delimiter '.' std::vector host_tokens; // port number as string, or nullopt if port missing std::optional port; }; /// \brief Class handling URLs using libcurl API. /// As with libcurl, only limited checks are performed in order to parse the /// required fields for a given URL string. class CurlURLHandle { public: CurlURLHandle() noexcept = default; ~CurlURLHandle() noexcept = default; // prohibit moves & copies CurlURLHandle(CurlURLHandle const&) = delete; CurlURLHandle(CurlURLHandle&& other) = delete; auto operator=(CurlURLHandle const&) = delete; auto operator=(CurlURLHandle&& other) = delete; /// \brief Creates a CurlURLHandle object by parsing the given URL. /// It performs also a normalization step of the path. Requires the protocol /// to be explicitly specified, i.e., it must have a non-empty scheme field. /// \returns Pointer to created object, nullptr on failure to parse, or /// nullopt on an unexpected exception. [[nodiscard]] auto static Create(std::string const& url) noexcept -> std::optional; /// \brief Creates a CurlURLHandle object by parsing the given URL. /// It allows the user to be very permissive with the types of URL strings /// it can parse by providing configuration arguments that mirror those /// provided by the libcurl API (see libcurl docs for effects of each flag). /// \param [in] ignore_fatal Do not log failure if error or exception. /// \returns Pointer to created object, nullptr on failure to parse with /// given arguments, or nullopt on an unexpected exception. [[nodiscard]] auto static CreatePermissive( std::string const& url, bool use_guess_scheme = false, bool use_default_scheme = false, bool use_non_support_scheme = false, bool use_no_authority = false, bool use_path_as_is = false, bool use_allow_space = false, bool ignore_fatal = false) noexcept -> std::optional; /// \brief Creates a duplicate CurlURLHandle object. /// \returns Pointer to duplicated object, or nullptr on errors. [[nodiscard]] auto Duplicate() noexcept -> CurlURLHandlePtr; /// \brief Recomposes the URL from the fields in the stored handle. /// Flags parallel the libcurl API for handling the scheme and port fields. /// \param [in] ignore_fatal Do not log failure if error or exception. /// \returns The recomposed URL as a string, or nullopt on errors. [[nodiscard]] auto GetURL(bool use_default_port = false, bool use_default_scheme = false, bool use_no_default_port = false, bool ignore_fatal = false) noexcept -> std::optional; /// \brief Gets the parsed scheme field. /// \returns Nullopt on errors, or an OptionalString containing either the /// existing stored scheme or nullopt if scheme field is missing. [[nodiscard]] auto GetScheme(bool use_default_scheme = false) noexcept -> std::optional; /// \brief While libcurl's URL API correctly checks that valid hostnames /// don't contain special characters, gitconfig key URLs (*..*) allow /// asterisks ('*'). This function recognizes such hostnames and returns a /// struct containing all the relevant parsed fields required for matching. /// \returns Pointer to said struct, nullopt if errors, nullptr if /// unparsable. [[nodiscard]] auto static ParseConfigKey(std::string const& key) noexcept -> std::optional; /// \brief Parses a given gitconfig key url component (e.g., http..*) /// and returns to what degree it matches the stored URL. /// In particular, a non-parsable key returns a non-match. /// \returns Matching degree struct, or nullopt on errors. [[nodiscard]] auto MatchConfigKey(std::string const& key) noexcept -> std::optional; /// \brief Checks if the stored URL matches a given "no_proxy"-style string. /// \returns Whether a match was found, or nullopt on errors. [[nodiscard]] auto NoproxyStringMatches( std::string const& no_proxy) noexcept -> std::optional; /// \brief Gets the hostname from URL. /// \returns The host name or std::nullopt if missing or on errors. [[nodiscard]] static auto GetHostname(std::string const& url) noexcept -> std::optional; private: // IMPORTANT: the CurlContext must be initialized before any curl // object! CurlContext curl_context_; std::unique_ptr handle_{nullptr, curl_url_closer}; /// \brief Try to parse the given key as a valid URL and, if successful, /// populate a struct with the parsed components needed for config matching. [[nodiscard]] auto static GetConfigStructFromKey( std::string const& key) noexcept -> std::optional; }; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_URL_HANDLE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_archive.cpp000066400000000000000000000210321516554100600274240ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/parse_archive.hpp" #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/utils/cpp/path.hpp" auto ParseArchiveContent(ExpressionPtr const& repo_desc, std::string const& origin) -> expected { // enforce mandatory fields auto repo_desc_content = repo_desc->At("content"); if (not repo_desc_content) { return unexpected{ "Mandatory field \"content\" is missing"}; } if (not repo_desc_content->get()->IsString()) { return unexpected{ fmt::format("Unsupported value {} for mandatory field \"content\"", repo_desc_content->get()->ToString())}; } auto const repo_desc_hash_info = HashInfo::Create(HashFunction::Type::GitSHA1, repo_desc_content->get()->String(), /*is_tree=*/false); if (not repo_desc_hash_info) { return unexpected{fmt::format( "Unsupported value {} for mandatory field \"content\"\n{}", repo_desc_content->get()->ToString(), repo_desc_hash_info.error())}; } auto repo_desc_fetch = repo_desc->At("fetch"); if (not repo_desc_fetch) { return unexpected{"Mandatory field \"fetch\" is missing"}; } if (not repo_desc_fetch->get()->IsString()) { return unexpected{ fmt::format("Unsupported value {} for mandatory field \"fetch\"", repo_desc_fetch->get()->ToString())}; } auto repo_desc_distfile = repo_desc->Get("distfile", Expression::none_t{}); auto repo_desc_sha256 = repo_desc->Get("sha256", Expression::none_t{}); auto repo_desc_sha512 = repo_desc->Get("sha512", Expression::none_t{}); // check optional mirrors auto repo_desc_mirrors = repo_desc->Get("mirrors", Expression::list_t{}); std::vector mirrors{}; if (repo_desc_mirrors->IsList()) { mirrors.reserve(repo_desc_mirrors->List().size()); for (auto const& elem : repo_desc_mirrors->List()) { if (not elem->IsString()) { return unexpected{fmt::format( "Unsupported list entry {} in optional field \"mirrors\"", elem->ToString())}; } mirrors.emplace_back(elem->String()); } } else { return unexpected{ fmt::format("Optional field \"mirrors\" should be a list of " "strings, but found: {}", repo_desc_mirrors->ToString())}; } return ArchiveContent{ .content_hash = *repo_desc_hash_info, .distfile = repo_desc_distfile->IsString() ? std::make_optional(repo_desc_distfile->String()) : std::nullopt, .fetch_url = repo_desc_fetch->get()->String(), .mirrors = std::move(mirrors), .sha256 = repo_desc_sha256->IsString() ? std::make_optional(repo_desc_sha256->String()) : std::nullopt, .sha512 = repo_desc_sha512->IsString() ? std::make_optional(repo_desc_sha512->String()) : std::nullopt, .origin = origin}; } auto ParseArchiveDescription(ExpressionPtr const& repo_desc, std::string const& repo_type, std::string const& origin, const AsyncMapConsumerLoggerPtr& logger) -> std::optional { auto const archive_content = ParseArchiveContent(repo_desc, origin); if (not archive_content) { (*logger)(fmt::format("ArchiveCheckout: {}", archive_content.error()), /*fatal=*/true); return std::nullopt; } // additional mandatory fields auto repo_desc_subdir = repo_desc->Get("subdir", Expression::none_t{}); auto subdir = std::filesystem::path(repo_desc_subdir->IsString() ? repo_desc_subdir->String() : "") .lexically_normal(); if (not PathIsNonUpwards(subdir)) { (*logger)(fmt::format("ArchiveCheckout: Expected field \"subdir\" to " "be a non-upwards path, but found {}", subdir.string()), /*fatal=*/true); return std::nullopt; } // check "special" pragma auto repo_desc_pragma = repo_desc->At("pragma"); bool const& pragma_is_map = repo_desc_pragma and repo_desc_pragma->get()->IsMap(); auto pragma_special = pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = pragma_special and pragma_special->get()->IsString() and kPragmaSpecialMap.contains(pragma_special->get()->String()) ? std::make_optional( kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // check "absent" pragma auto pragma_absent = pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt; auto pragma_absent_value = pragma_absent and pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); return ArchiveRepoInfo{.archive = *archive_content, .repo_type = repo_type, .subdir = subdir.empty() ? "." : subdir.string(), .pragma_special = pragma_special_value, .absent = pragma_absent_value}; } auto ParseForeignFileDescription(ExpressionPtr const& repo_desc, std::string const& origin, const AsyncMapConsumerLoggerPtr& logger) -> std::optional { auto const archive_content = ParseArchiveContent(repo_desc, origin); if (not archive_content) { (*logger)(archive_content.error(), /*fatal=*/true); return std::nullopt; } auto name = repo_desc->At("name"); if (not name) { (*logger)( "Mandatory field \"name\" for foreign file repository is missing", true); return std::nullopt; } if (not name->get()->IsString()) { (*logger)(fmt::format("Field \"name\" has to be a file name, given as " "string, but found {}", name->get()->ToString()), true); return std::nullopt; } if (not IsValidFileName(name->get()->String())) { (*logger)(fmt::format("Field \"name\" has to be valid a file name, but " "found {}", name->get()->ToString()), true); return std::nullopt; } auto executable = repo_desc->Get("executable", Expression::kFalse); if (not executable->IsBool()) { (*logger)(fmt::format( "Field \"executable\" has to be a boolean, but found {}", executable->ToString()), true); return std::nullopt; } bool absent{}; auto pragma = repo_desc->Get("pragma", Expression::kEmptyMap); if (pragma->IsMap()) { auto pragma_absent = pragma->Get("absent", Expression::kFalse); if (pragma_absent->IsBool()) { absent = pragma_absent->Bool(); } } return ForeignFileInfo{.archive = *archive_content, .name = name->get()->String(), .executable = executable->Bool(), .absent = absent}; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_archive.hpp000066400000000000000000000037741516554100600274460ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_ARCHIVE_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_ARCHIVE_HPP #include #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/utils/cpp/expected.hpp" auto ParseArchiveContent(ExpressionPtr const& repo_desc, std::string const& origin) -> expected; // Parse the description of an archive repository; if an error // occurs, call the logger with fatal set to true and return std::nullopt // instead. auto ParseArchiveDescription(ExpressionPtr const& repo_desc, std::string const& repo_type, std::string const& origin, const AsyncMapConsumerLoggerPtr& logger) -> std::optional; // Parse the description of a foreign-file repository; if an error // occurs, call the logger with fatal set to true and return std::nullopt // instead. auto ParseForeignFileDescription(ExpressionPtr const& repo_desc, std::string const& origin, const AsyncMapConsumerLoggerPtr& logger) -> std::optional; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_ARCHIVE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_git_tree.cpp000066400000000000000000000111501516554100600276050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/parse_git_tree.hpp" #include #include #include // std::move #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" [[nodiscard]] auto ParseGitTree(ExpressionPtr const& repo_desc, std::optional origin) -> expected { auto repo_desc_hash = repo_desc->At("id"); if (not repo_desc_hash) { return unexpected{"Mandatory field \"id\" is missing"}; } if (not repo_desc_hash->get()->IsString()) { return unexpected{ fmt::format("Unsupported value {} for " "mandatory field \"id\"", repo_desc_hash->get()->ToString())}; } auto repo_desc_hash_info = HashInfo::Create(HashFunction::Type::GitSHA1, repo_desc_hash->get()->String(), /*is_tree=*/true); if (not repo_desc_hash_info) { return unexpected{ fmt::format("Unsupported value {} for " "mandatory field \"id\"\n{}", repo_desc_hash->get()->ToString(), std::move(repo_desc_hash_info).error())}; } auto repo_desc_cmd = repo_desc->At("cmd"); if (not repo_desc_cmd) { return unexpected{"Mandatory field \"cmd\" is missing"}; } if (not repo_desc_cmd->get()->IsList()) { return unexpected{ fmt::format("Unsupported value {} for " "mandatory field \"cmd\"", repo_desc_cmd->get()->ToString())}; } std::vector cmd{}; for (auto const& token : repo_desc_cmd->get()->List()) { if (token.IsNotNull() and token->IsString()) { cmd.emplace_back(token->String()); } else { return unexpected{ fmt::format("Unsupported entry {} " "in mandatory field \"cmd\"", token->ToString())}; } } std::map env{}; auto repo_desc_env = repo_desc->Get("env", Expression::none_t{}); if (repo_desc_env.IsNotNull() and repo_desc_env->IsMap()) { for (auto const& envar : repo_desc_env->Map().Items()) { if (envar.second.IsNotNull() and envar.second->IsString()) { env.insert({envar.first, envar.second->String()}); } else { return unexpected{ fmt::format("Unsupported value {} for " "key {} in optional field \"envs\"", envar.second->ToString(), nlohmann::json(envar.first).dump())}; } } } std::vector inherit_env{}; auto repo_desc_inherit_env = repo_desc->Get("inherit env", Expression::none_t{}); if (repo_desc_inherit_env.IsNotNull() and repo_desc_inherit_env->IsList()) { for (auto const& envvar : repo_desc_inherit_env->List()) { if (envvar->IsString()) { inherit_env.emplace_back(envvar->String()); } else { return unexpected{ fmt::format("Not a variable " "name in the specification " "of \"inherit env\": {}", envvar->ToString())}; } } } // populate struct auto info = GitTreeInfo{.tree_hash = *std::move(repo_desc_hash_info), .env_vars = std::move(env), .inherit_env = std::move(inherit_env), .command = std::move(cmd), .origin = origin ? std::move(*origin) : std::string{}}; return info; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_git_tree.hpp000066400000000000000000000022231516554100600276130ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_GIT_TREE_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_GIT_TREE_HPP #include #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/utils/cpp/expected.hpp" [[nodiscard]] auto ParseGitTree( ExpressionPtr const& repo_desc, std::optional origin = std::nullopt) -> expected; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_GIT_TREE_HPP just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_precomputed_root.cpp000066400000000000000000000126641516554100600314100ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/utils/parse_precomputed_root.hpp" #include #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" namespace { [[nodiscard]] auto ParseAbsent(ExpressionPtr const& repository) -> expected { auto const pragma = repository->Get("pragma", Expression::none_t{}); if (not pragma.IsNotNull()) { // Missing "pragma", absent == false return false; } if (not pragma->IsMap()) { return unexpected{fmt::format( "Key \"pragma\", if given, should be a map, but found {}", pragma->ToString())}; } auto const is_absent = pragma->Get("absent", Expression::none_t{}); if (not is_absent.IsNotNull()) { // "pragma" doesn't contain "absent", absent == false return false; } if (not is_absent->IsBool()) { return unexpected{fmt::format( "Expected pragma \"absent\" to be boolean, but found {}", is_absent->ToString())}; } return is_absent->Bool(); } [[nodiscard]] auto ParseComputedRoot(ExpressionPtr const& repository) -> expected { auto const repo = repository->Get("repo", Expression::none_t{}); if (not repo.IsNotNull()) { return unexpected{"Mandatory key \"repo\" is missing"}; } if (not repo->IsString()) { return unexpected{fmt::format("Unsupported value for key \"repo\":\n{}", repo->ToString())}; } auto const target = repository->Get("target", Expression::none_t{}); if (not target.IsNotNull()) { return unexpected{"Mandatory key \"target\" is missing"}; } if (not target->IsList() or target->List().size() != 2) { return unexpected{fmt::format( "Unsupported value for key \"target\":\n{}", target->ToString())}; } auto const& target_list = target->List(); auto const target_module = target_list.at(0); auto const target_name = target_list.at(1); if (not target_module->IsString() or not target_name->IsString()) { return unexpected{fmt::format( "Unsupported format for key \"target\":\n{}", target->ToString())}; } auto const config = repository->Get("config", Expression::none_t{}); if (config.IsNotNull() and not config->IsMap()) { return unexpected{fmt::format( "Unsupported value for key \"config\":\n{}", config->ToString())}; } auto absent = ParseAbsent(repository); if (not absent.has_value()) { return unexpected{std::move(absent).error()}; } return ComputedRoot{.repository = repo->String(), .target_module = target_module->String(), .target_name = target_module->String(), .config = config.IsNotNull() ? config->ToJson() : nlohmann::json::object(), .absent = *absent}; } [[nodiscard]] auto ParseTreeStructureRoot(ExpressionPtr const& repository) -> expected { auto const repo = repository->Get("repo", Expression::none_t{}); if (not repo.IsNotNull()) { return unexpected{"Mandatory key \"repo\" is missing"}; } auto absent = ParseAbsent(repository); if (not absent.has_value()) { return unexpected{std::move(absent).error()}; } return TreeStructureRoot{.repository = repo->String(), .absent = *absent}; } } // namespace auto ParsePrecomputedRoot(ExpressionPtr const& repository) -> expected { if (not repository.IsNotNull() or not repository->IsMap()) { return unexpected{"Repository has an incorrect format"}; } auto const type = repository->Get("type", Expression::none_t()); if (not type.IsNotNull()) { return unexpected{"Mandatory key \"type\" is missing"}; } if (not type->IsString()) { return unexpected{fmt::format("Unsupported value for key \"type\":\n{}", type->ToString())}; } auto const& type_marker = type->String(); if (type_marker == ComputedRoot::kMarker) { auto computed = ParseComputedRoot(repository); if (not computed) { return unexpected{std::move(computed).error()}; } return PrecomputedRoot{*std::move(computed)}; } if (type_marker == TreeStructureRoot::kMarker) { auto tree_structure = ParseTreeStructureRoot(repository); if (not tree_structure) { return unexpected{std::move(tree_structure).error()}; } return PrecomputedRoot{*std::move(tree_structure)}; } return unexpected{ fmt::format("Unknown type {} of precomputed repository", type_marker)}; } just-buildsystem-justbuild-b1fb5fa/src/other_tools/utils/parse_precomputed_root.hpp000066400000000000000000000021501516554100600314020ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_PRECOMPUTED_ROOT_HPP #define INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_PRECOMPUTED_ROOT_HPP #include #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/utils/cpp/expected.hpp" [[nodiscard]] auto ParsePrecomputedRoot(ExpressionPtr const& repository) -> expected; #endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_PRECOMPUTED_ROOT_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/000077500000000000000000000000001516554100600215465ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/utils/archive/000077500000000000000000000000001516554100600231675ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/utils/archive/TARGETS000066400000000000000000000006161516554100600242260ustar00rootroot00000000000000{ "archive_ops": { "type": ["@", "rules", "CC", "library"] , "name": ["archive_ops"] , "hdrs": ["archive_ops.hpp"] , "srcs": ["archive_ops.cpp"] , "stage": ["src", "utils", "archive"] , "private-deps": [ ["", "libarchive"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } } just-buildsystem-justbuild-b1fb5fa/src/utils/archive/archive_ops.cpp000066400000000000000000000370721516554100600262060ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/archive/archive_ops.hpp" #include #include #include #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" extern "C" { #include #include } #ifndef BOOTSTRAP_BUILD_TOOL namespace { /// \brief Default block size for archive extraction. constexpr std::size_t kArchiveBlockSize = 10240; /// \brief Clean-up function for archive entry objects. void archive_entry_cleanup(archive_entry* entry) { if (entry != nullptr) { archive_entry_free(entry); } } /// \brief Clean-up function for archive objects open for writing. void archive_write_closer(archive* a_out) { if (a_out != nullptr) { archive_write_close(a_out); // libarchive handles non-openness archive_write_free(a_out); // also do cleanup! } } /// \brief Clean-up function for archive objects open for reading. void archive_read_closer(archive* a_in) { if (a_in != nullptr) { archive_read_close(a_in); // libarchive handles non-openness archive_read_free(a_in); // also do cleanup! } } auto enable_write_filter(archive* aw, ArchiveType type) -> bool { switch (type) { case ArchiveType::Tar: return true; // no compression filter case ArchiveType::TarGz: return (archive_write_add_filter_gzip(aw) == ARCHIVE_OK); case ArchiveType::TarBz2: return (archive_write_add_filter_bzip2(aw) == ARCHIVE_OK); case ArchiveType::TarXz: return (archive_write_add_filter_xz(aw) == ARCHIVE_OK); case ArchiveType::TarLz: return (archive_write_add_filter_lzip(aw) == ARCHIVE_OK); case ArchiveType::TarLzma: return (archive_write_add_filter_lzma(aw) == ARCHIVE_OK); default: return false; } } auto enable_read_filter(archive* ar, ArchiveType type) -> bool { switch (type) { case ArchiveType::Tar: return true; // no outside compression filter case ArchiveType::TarGz: return (archive_read_support_filter_gzip(ar) == ARCHIVE_OK); case ArchiveType::TarBz2: return (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); case ArchiveType::TarXz: return (archive_read_support_filter_xz(ar) == ARCHIVE_OK); case ArchiveType::TarLz: return (archive_read_support_filter_lzip(ar) == ARCHIVE_OK); case ArchiveType::TarLzma: return (archive_read_support_filter_lzma(ar) == ARCHIVE_OK); case ArchiveType::TarAuto: return (archive_read_support_filter_all(ar) == ARCHIVE_OK); default: return false; } } } // namespace #endif // BOOTSTRAP_BUILD_TOOL auto ArchiveOps::WriteEntry(archive_entry* entry, archive* aw) -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL std::filesystem::path entry_path{archive_entry_sourcepath(entry)}; // only write to archive if entry is file if (FileSystemManager::IsFile(entry_path)) { auto content = FileSystemManager::ReadFile(entry_path); if (not content) { return "ArchiveOps: failed to open file entry while creating " "archive"; } if (not content->empty()) { auto content_size = content->size(); archive_write_data(aw, content->c_str(), content_size); } } #endif // BOOTSTRAP_BUILD_TOOL return std::nullopt; } auto ArchiveOps::CopyData(archive* ar, archive* aw) -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL int r{}; const void* buff{nullptr}; std::size_t size{}; la_int64_t offset{}; while (true) { r = archive_read_data_block(ar, &buff, &size, &offset); if (r == ARCHIVE_EOF) { return std::nullopt; // success! } if (r != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } if (archive_write_data_block(aw, buff, size, offset) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(aw)); } } #endif // BOOTSTRAP_BUILD_TOOL return std::nullopt; // success! } auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL switch (type) { case ArchiveType::Zip: { if (archive_write_set_format_zip(aw) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(aw)); } } break; case ArchiveType::_7Zip: { if (archive_write_set_format_7zip(aw) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(aw)); } } break; case ArchiveType::ZipAuto: { return std::string( "ArchiveOps: Writing a zip-like archive must be explicit"); } case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: case ArchiveType::TarXz: case ArchiveType::TarLz: case ArchiveType::TarLzma: { if ((archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) or not enable_write_filter(aw, type)) { return std::string("ArchiveOps: ") + std::string(archive_error_string(aw)); } } break; case ArchiveType::TarAuto: return std::string( "ArchiveOps: Writing a tarball-type archive must be explicit!"); } #endif // BOOTSTRAP_BUILD_TOOL return std::nullopt; // success! } auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) -> std::optional { #ifndef BOOTSTRAP_BUILD_TOOL switch (type) { case ArchiveType::Zip: { if (archive_read_support_format_zip(ar) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } } break; case ArchiveType::_7Zip: { if (archive_read_support_format_7zip(ar) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } } break; case ArchiveType::ZipAuto: { if (archive_read_support_format_7zip(ar) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } if (archive_read_support_format_zip(ar) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } } break; case ArchiveType::TarAuto: case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: case ArchiveType::TarXz: case ArchiveType::TarLz: case ArchiveType::TarLzma: { if ((archive_read_support_format_tar(ar) != ARCHIVE_OK) or not enable_read_filter(ar, type)) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } } break; } #endif // BOOTSTRAP_BUILD_TOOL return std::nullopt; // success! } auto ArchiveOps::CreateArchive(ArchiveType type, std::string const& name, std::filesystem::path const& source) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #endif return CreateArchive(type, name, source, std::filesystem::path(".")); } auto ArchiveOps::CreateArchive(ArchiveType type, std::string const& name, std::filesystem::path const& source, std::filesystem::path const& destDir) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { // make sure paths will be relative wrt current dir auto rel_source = std::filesystem::relative(source); std::unique_ptr a_out{ archive_write_new(), archive_write_closer}; if (a_out == nullptr) { return std::string("ArchiveOps: archive_write_new failed"); } // enable the correct format for archive type auto res = EnableWriteFormats(a_out.get(), type); if (res != std::nullopt) { return res; } // open archive to write if (not FileSystemManager::CreateDirectory(destDir)) { return std::string( "ArchiveOps: could not create destination directory ") + destDir.string(); } if (archive_write_open_filename( a_out.get(), (destDir / std::filesystem::path(name)).c_str()) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(a_out.get())); } // open source std::unique_ptr disk{ archive_read_disk_new(), archive_read_closer}; if (disk == nullptr) { return std::string("ArchiveOps: archive_read_disk_new failed"); } archive_read_disk_set_standard_lookup(disk.get()); if (archive_read_disk_open(disk.get(), rel_source.c_str()) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(disk.get())); } // create archive while (true) { std::unique_ptr entry{archive_entry_new(), archive_entry_cleanup}; if (entry == nullptr) { return std::string("ArchiveOps: archive_entry_new failed"); } int r = archive_read_next_header2(disk.get(), entry.get()); if (r == ARCHIVE_EOF) { return std::nullopt; // nothing left to archive; success! } if (r != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(disk.get())); } // if entry is a directory, make sure we descend into all its // children archive_read_disk_descend(disk.get()); // get info on current entry if (archive_write_header(a_out.get(), entry.get()) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(a_out.get())); } // write entry into archive auto res = WriteEntry(entry.get(), a_out.get()); if (res != std::nullopt) { return res; } } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "archive create failed with:\n{}", ex.what()); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } auto ArchiveOps::ExtractArchive(ArchiveType type, std::filesystem::path const& source) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #endif return ExtractArchive(type, source, std::filesystem::path(".")); } auto ArchiveOps::ExtractArchive(ArchiveType type, std::filesystem::path const& source, std::filesystem::path const& destDir) noexcept -> std::optional { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; #else try { std::unique_ptr a_in{ archive_read_new(), archive_read_closer}; if (a_in == nullptr) { return std::string("ArchiveOps: archive_read_new failed"); } // enable support for known formats auto res = EnableReadFormats(a_in.get(), type); if (res != std::nullopt) { return res; } // open archive for reading if (archive_read_open_filename( a_in.get(), source.c_str(), kArchiveBlockSize) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(a_in.get())); } // set up writer to disk std::unique_ptr disk{ archive_write_disk_new(), archive_write_closer}; if (disk == nullptr) { return std::string("ArchiveOps: archive_write_disk_new failed"); } // Select which attributes we want to restore. uint flags = ARCHIVE_EXTRACT_TIME; flags |= static_cast(ARCHIVE_EXTRACT_PERM); flags |= static_cast(ARCHIVE_EXTRACT_FFLAGS); archive_write_disk_set_options(disk.get(), static_cast(flags)); archive_write_disk_set_standard_lookup(disk.get()); // make sure destination directory exists if (not FileSystemManager::CreateDirectory(destDir)) { return std::string( "ArchiveOps: could not create destination directory ") + destDir.string(); } // extract the archive archive_entry* entry{nullptr}; while (true) { int r = archive_read_next_header(a_in.get(), &entry); if (r == ARCHIVE_EOF) { return std::nullopt; // nothing left to extract; success! } if (r != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(a_in.get())); } // set correct destination path auto new_entry_path = destDir / std::filesystem::path(archive_entry_pathname(entry)); archive_entry_set_pathname(entry, new_entry_path.c_str()); if (archive_write_header(disk.get(), entry) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(disk.get())); } // write to disk if file if (archive_entry_size(entry) > 0) { auto res = CopyData(a_in.get(), disk.get()); if (res != std::nullopt) { return res; } } // finish entry writing if (archive_write_finish_entry(disk.get()) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(disk.get())); } } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "archive extract failed with:\n{}", ex.what()); return std::nullopt; } #endif // BOOTSTRAP_BUILD_TOOL } just-buildsystem-justbuild-b1fb5fa/src/utils/archive/archive_ops.hpp000066400000000000000000000103141516554100600262010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_ARCHIVE_ARCHIVE_OPS_HPP #define INCLUDED_SRC_UTILS_ARCHIVE_ARCHIVE_OPS_HPP #include #include #include #include extern "C" { using archive = struct archive; using archive_entry = struct archive_entry; } enum class ArchiveType : std::uint8_t { Zip, _7Zip, ZipAuto, // autodetect zip-like archives Tar, // uncompressed TarGz, TarBz2, TarXz, TarLz, TarLzma, TarAuto // autodetect tarball-type archives }; /// \brief Class handling archiving and unarchiving operations via libarchive class ArchiveOps { public: /// \brief Create archive of given type from file or directory at /// source. All paths will be takes relative to current directory. /// Destination folder is the current directory. Archive is stored under /// given name. Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static CreateArchive( ArchiveType type, std::string const& name, std::filesystem::path const& source) noexcept -> std::optional; /// \brief Create archive of given type from file or directory at source and /// store it in destDir folder under given name. All paths will be taken as /// relative to the current directory. Destination directory is created if /// not present. Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static CreateArchive( ArchiveType type, std::string const& name, std::filesystem::path const& source, std::filesystem::path const& destDir) noexcept -> std::optional; /// \brief Extract archive pointed to by source into destDir folder. The /// destination folder is the current directory and the type of archive is /// specified from currently supported formats: tar, zip, tar.gz, tar.bz2. /// Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static ExtractArchive( ArchiveType type, std::filesystem::path const& source) noexcept -> std::optional; /// \brief Extract archive pointed to by source into destDir folder. The /// type of archive is specified from currently supported formats: tar, zip, /// tar.gz, tar.bz2. Returns nullopt on success, or an error string if /// failure. [[nodiscard]] auto static ExtractArchive( ArchiveType type, std::filesystem::path const& source, std::filesystem::path const& destDir) noexcept -> std::optional; private: /// \brief Copy entry into archive object. /// Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static WriteEntry(archive_entry* entry, archive* aw) -> std::optional; /// \brief Copy data blocks from one archive object to another. /// Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static CopyData(archive* ar, archive* aw) -> std::optional; /// \brief Set up the appropriate supported format for writing an archive. /// Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static EnableWriteFormats(archive* aw, ArchiveType type) -> std::optional; /// \brief Set up the supported formats for reading in an archive. /// Returns nullopt on success, or an error string if failure. [[nodiscard]] auto static EnableReadFormats(archive* ar, ArchiveType type) -> std::optional; }; #endif // INCLUDED_SRC_UTILS_ARCHIVE_ARCHIVE_OPS_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/automata/000077500000000000000000000000001516554100600233615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/utils/automata/TARGETS000066400000000000000000000005151516554100600244160ustar00rootroot00000000000000{ "dfa_minimizer": { "type": ["@", "rules", "CC", "library"] , "name": ["dfa_minimizer"] , "hdrs": ["dfa_minimizer.hpp"] , "stage": ["src", "utils", "automata"] , "deps": [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "hex_string"] ] } } just-buildsystem-justbuild-b1fb5fa/src/utils/automata/dfa_minimizer.hpp000066400000000000000000000214341516554100600267130ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_AUTOMATA_DFA_MINIMIZER_HPP #define INCLUDED_SRC_UTILS_AUTOMATA_DFA_MINIMIZER_HPP #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/hex_string.hpp" // Minimizes a DFA by separating distinguishable states. States added with same // content id are considered initially indistinguishable. The algorithm has // complexity O(n^2) among each set of initially indistinguishable states. // Note that for incomplete graphs, two states are considered distinguishable if // they transition for the same symbol to two differently named non-existing // states. This is done for efficiency reasons, as we then can avoid creating an // additional bucket for non-existing states. This is sufficient for our use // case, as we are only interested in the bisimulation of states in complete // graphs. class DFAMinimizer { // Maps symbols to states using transitions_t = std::map; using states_t = std::unordered_map; // Bucket of states with equal local properties (content and acceptance) struct Bucket { std::vector symbols; states_t states{}; }; // Key used for state pairs. Reordering names will result in the same key. class StatePairKey { public: struct Hash { [[nodiscard]] auto operator()(StatePairKey const& p) const -> std::size_t { std::size_t hash{}; hash_combine(&hash, p.Names().first); hash_combine(&hash, p.Names().second); return hash; } }; StatePairKey(std::string first, std::string second) { if (first < second) { data_ = std::make_pair(std::move(first), std::move(second)); } else { data_ = std::make_pair(std::move(second), std::move(first)); } } [[nodiscard]] auto Names() const& -> std::pair const& { return data_; } [[nodiscard]] auto operator==(StatePairKey const& other) const -> bool = default; [[nodiscard]] auto operator!=(StatePairKey const& other) const -> bool = default; private: std::pair data_; }; // Value of state pairs. struct StatePairValue { // Parent pairs depending on this pair's distinguishability std::vector parents; // Distinguishability flag (true means distinguishable) bool marked{}; }; using state_pairs_t = std::unordered_map; public: using bisimulation_t = std::unordered_map; // Add state with name, transitions, and content id. States with the same // content id are initially considered indistinguishable. void AddState(std::string const& name, transitions_t const& transitions, std::string const& content_id = {}) { auto symbols = GetKeys(transitions); // Compute bucket id from content id and transition symbols auto bucket_id = nlohmann::json(std::pair{ToHexString(content_id), symbols}).dump(); // Store indistinguishable states in same bucket auto it = buckets_.find(bucket_id); if (it == buckets_.end()) { it = buckets_.emplace(bucket_id, Bucket{std::move(symbols)}).first; } it->second.states.emplace(name, transitions); Ensures(buckets_by_state_.emplace(name, bucket_id).second); } // Compute bisimulation for each state and return a map, which maps a // state name to its bisimulation if one was found. [[nodiscard]] auto ComputeBisimulation() const -> bisimulation_t { auto pairs = CreatePairs(); // Mark pairs of distinguishable states for (auto& [key, ab_val] : pairs) { auto const& [a, b] = key.Names(); auto const& bucket_id = buckets_by_state_.at(a); auto const& bucket = buckets_.at(bucket_id); for (auto const& sym : bucket.symbols) { auto const& r = bucket.states.at(a).at(sym); auto const& s = bucket.states.at(b).at(sym); if (r != s) { auto* rs_val = LookupPairValue(&pairs, {r, s}); if (rs_val == nullptr or rs_val->marked) { // if rs_val == nullptr, the pair is missing, due to: // - both do not exist (and are named differently) // - either r or s does not exist // - r and s do exist but are not in the same bucket MarkPairValue(&ab_val); } else if (rs_val != nullptr) { // Remember ab_val if rs_val ever gets marked rs_val->parents.emplace_back(&ab_val); } } } } // Compute the bisimulation for each state bisimulation_t bisimulation{}; for (auto const& [_, bucket] : buckets_) { // Consider states in unmarked pairs to be equivalent auto all_states = GetKeys(bucket.states); while (not all_states.empty()) { auto last = std::move(all_states.back()); all_states.pop_back(); auto unequal_states = all_states; for (auto const& state : all_states) { // Lookup is safe, both must be in the same bucket and // therefore the pair is are guaranteed to exist if (not LookupPairValue(&pairs, {last, state})->marked) { bisimulation.emplace(state, last); std::erase(unequal_states, state); } } all_states = unequal_states; } } return bisimulation; } private: std::unordered_map buckets_; std::unordered_map buckets_by_state_; template [[nodiscard]] static auto GetKeys(M const& map) -> std::vector { auto keys = std::vector{}; keys.reserve(map.size()); std::transform(map.begin(), map.end(), std::back_inserter(keys), [](auto const& kv) { return kv.first; }); return keys; } // Returns nullptr if no pair with given key was found. [[nodiscard]] static auto LookupPairValue( gsl::not_null const& pairs, StatePairKey const& key) -> StatePairValue* { auto it = pairs->find(key); if (it != pairs->end()) { return &it->second; } return nullptr; } // Mark pair as distinguishable and recursively mark all parents. static void MarkPairValue(gsl::not_null const& data) { data->marked = true; for (auto* parent : data->parents) { if (not parent->marked) { MarkPairValue(parent); } } } // Create n to n pairs for all states in the same bucket. [[nodiscard]] auto CreatePairs() const -> state_pairs_t { state_pairs_t pairs{}; for (auto const& [_, bucket] : buckets_) { auto const& states = bucket.states; auto const n = states.size(); pairs.reserve(pairs.size() + ((n * (n - 1)) / 2)); auto const end = states.end(); for (auto it = ++states.begin(); it != end; ++it) { for (auto it2 = states.begin(); it2 != it; ++it2) { pairs.emplace(StatePairKey{it->first, it2->first}, StatePairValue{}); } } } return pairs; } }; #endif // INCLUDED_SRC_UTILS_AUTOMATA_DFA_MINIMIZER_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/000077500000000000000000000000001516554100600223305ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/TARGETS000066400000000000000000000072021516554100600233650ustar00rootroot00000000000000{ "hash_combine": { "type": ["@", "rules", "CC", "library"] , "name": ["hash_combine"] , "hdrs": ["hash_combine.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "utils", "cpp"] } , "type_safe_arithmetic": { "type": ["@", "rules", "CC", "library"] , "name": ["type_safe_arithmetic"] , "hdrs": ["type_safe_arithmetic.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "utils", "cpp"] } , "json": { "type": ["@", "rules", "CC", "library"] , "name": ["json"] , "hdrs": ["json.hpp"] , "deps": ["gsl", ["@", "fmt", "", "fmt"], ["@", "json", "", "json"]] , "stage": ["src", "utils", "cpp"] } , "concepts": { "type": ["@", "rules", "CC", "library"] , "name": ["concepts"] , "hdrs": ["concepts.hpp"] , "stage": ["src", "utils", "cpp"] } , "atomic": { "type": ["@", "rules", "CC", "library"] , "name": ["atomic"] , "hdrs": ["atomic.hpp"] , "stage": ["src", "utils", "cpp"] } , "hex_string": { "type": ["@", "rules", "CC", "library"] , "name": ["hex_string"] , "hdrs": ["hex_string.hpp"] , "stage": ["src", "utils", "cpp"] } , "path": { "type": ["@", "rules", "CC", "library"] , "name": ["path"] , "hdrs": ["path.hpp"] , "stage": ["src", "utils", "cpp"] } , "path_rebase": { "type": ["@", "rules", "CC", "library"] , "name": ["path_rebase"] , "hdrs": ["path_rebase.hpp"] , "stage": ["src", "utils", "cpp"] } , "vector": { "type": ["@", "rules", "CC", "library"] , "name": ["vector"] , "hdrs": ["vector.hpp"] , "stage": ["src", "utils", "cpp"] } , "tmp_dir": { "type": ["@", "rules", "CC", "library"] , "name": ["tmp_dir"] , "hdrs": ["tmp_dir.hpp"] , "srcs": ["tmp_dir.cpp"] , "stage": ["src", "utils", "cpp"] , "private-deps": [ ["@", "gsl", "", "gsl"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] } , "file_locking": { "type": ["@", "rules", "CC", "library"] , "name": ["file_locking"] , "hdrs": ["file_locking.hpp"] , "srcs": ["file_locking.cpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "utils", "cpp"] , "private-deps": [ ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "path"] ] } , "gsl": { "type": ["@", "rules", "CC", "library"] , "name": ["gsl"] , "hdrs": ["gsl.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "utils", "cpp"] } , "path_hash": { "type": ["@", "rules", "CC", "library"] , "name": ["path_hash"] , "hdrs": ["path_hash.hpp"] , "stage": ["src", "utils", "cpp"] } , "prefix": { "type": ["@", "rules", "CC", "library"] , "name": ["prefix"] , "hdrs": ["prefix.hpp"] , "stage": ["src", "utils", "cpp"] } , "expected": { "type": ["@", "rules", "CC", "library"] , "name": ["expected"] , "hdrs": ["expected.hpp"] , "stage": ["src", "utils", "cpp"] } , "back_map": { "type": ["@", "rules", "CC", "library"] , "name": ["back_map"] , "hdrs": ["back_map.hpp"] , "deps": [["@", "gsl", "", "gsl"]] , "stage": ["src", "utils", "cpp"] } , "in_place_visitor": { "type": ["@", "rules", "CC", "library"] , "name": ["in_place_visitor"] , "hdrs": ["in_place_visitor.hpp"] , "stage": ["src", "utils", "cpp"] } , "incremental_reader": { "type": ["@", "rules", "CC", "library"] , "name": ["incremental_reader"] , "hdrs": ["incremental_reader.hpp"] , "srcs": ["incremental_reader.cpp"] , "deps": ["expected", ["@", "gsl", "", "gsl"]] , "private-deps": ["in_place_visitor", ["@", "fmt", "", "fmt"]] , "stage": ["src", "utils", "cpp"] } } just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/atomic.hpp000066400000000000000000000115361516554100600243230ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP #define INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP #include #include #include #include //std::unique_lock #include #include // IWYU pragma: keep #include // IWYU pragma: keep // Atomic wrapper with notify/wait capabilities. // TODO(modernize): Replace any use this class by C++20's std::atomic, once // libcxx adds support for notify_*() and wait(). // [https://libcxx.llvm.org/Status/Cxx20.html] template class atomic { // NOLINT(readability-identifier-naming) public: atomic() = default; explicit atomic(T value) : value_{std::move(value)} {} atomic(atomic const& other) = delete; atomic(atomic&& other) = delete; ~atomic() = default; auto operator=(atomic const& other) -> atomic& = delete; auto operator=(atomic&& other) -> atomic& = delete; auto operator=(T desired) -> T { // NOLINT std::shared_lock lock(mutex_); value_ = desired; return desired; } operator T() const { return static_cast(value_); } // NOLINT void store(T desired, std::memory_order order = std::memory_order_seq_cst) { std::shared_lock lock(mutex_); value_.store(std::move(desired), order); } [[nodiscard]] auto load( std::memory_order order = std::memory_order_seq_cst) const -> T { return value_.load(order); } template requires(std::is_integral_v) auto operator++() -> T { std::shared_lock lock(mutex_); return ++value_; } template requires(std::is_integral_v) [[nodiscard]] auto operator++(int) -> T { std::shared_lock lock(mutex_); return value_++; } template requires(std::is_integral_v) auto operator--() -> T { std::shared_lock lock(mutex_); return --value_; } template requires(std::is_integral_v) [[nodiscard]] auto operator--(int) -> T { std::shared_lock lock(mutex_); return value_--; } void notify_one() { cv_.notify_one(); } void notify_all() { cv_.notify_all(); } void wait(T old, std::memory_order order = std::memory_order::seq_cst) const { std::unique_lock lock(mutex_); cv_.wait(lock, [this, &old, order]() { return value_.load(order) != old; }); } private: std::atomic value_{}; mutable std::shared_mutex mutex_; mutable std::condition_variable_any cv_; }; // Atomic shared_pointer with notify/wait capabilities. // TODO(modernize): Replace any use this class by C++20's // std::atomic>, once libcxx adds support for it. // [https://libcxx.llvm.org/Status/Cxx20.html] template class atomic_shared_ptr { // NOLINT(readability-identifier-naming) using ptr_t = std::shared_ptr; public: atomic_shared_ptr() = default; explicit atomic_shared_ptr(ptr_t value) : value_{std::move(value)} {} atomic_shared_ptr(atomic_shared_ptr const& other) = delete; atomic_shared_ptr(atomic_shared_ptr&& other) = delete; ~atomic_shared_ptr() = default; auto operator=(atomic_shared_ptr const& other) -> atomic_shared_ptr& = delete; auto operator=(atomic_shared_ptr&& other) -> atomic_shared_ptr& = delete; auto operator=(ptr_t desired) -> ptr_t { // NOLINT std::shared_lock lock(mutex_); std::atomic_store(&value_, desired); return desired; } operator ptr_t() const { return std::atomic_load(&value_); } // NOLINT void store(ptr_t desired) { std::shared_lock lock(mutex_); std::atomic_store(&value_, std::move(desired)); } [[nodiscard]] auto load() const -> ptr_t { return std::atomic_load(&value_); } void notify_one() { cv_.notify_one(); } void notify_all() { cv_.notify_all(); } void wait(ptr_t old) const { std::unique_lock lock(mutex_); cv_.wait(lock, [this, &old]() { return value_ != old; }); } private: ptr_t value_{}; mutable std::shared_mutex mutex_; mutable std::condition_variable_any cv_; }; #endif // INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/back_map.hpp000066400000000000000000000176331516554100600246100ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_BACK_MAP_HPP #define INCLUDED_SRC_UTILS_CPP_BACK_MAP_HPP #include #include #include #include #include #include #include #include // IWYU pragma: keep #include "gsl/gsl" /// \brief Backmap a container of Values to Keys using a given Converter, and /// provide an interface for quick lookup of values by keys. /// \tparam TKey Result of conversion. Stored by value, std::hash /// MUST be available. /// \tparam TValue Source data for conversion. Stored by pointer, the /// source container MUST stay alive. template class BackMap final { public: template using IsKeyWithError = std::bool_constant and std::is_convertible_v>; /// \brief Converter from TValue to TKey. The return type may be different /// from TKey, in such a case Converter is considered a function that DOES /// return TKey, but may fail and return std::nullopt or an unexpected. /// TResult must be either the same type as TKey, or support conversions /// from TKey and to bool. template requires(std::is_same_v or IsKeyWithError::value) using Converter = std::function; explicit BackMap() = default; BackMap(BackMap const&) = delete; BackMap(BackMap&&) = delete; auto operator=(BackMap const&) -> BackMap& = delete; auto operator=(BackMap&&) -> BackMap& = delete; ~BackMap() = default; /// \brief Create a BackMap by iterating over container and applying /// Converter. /// \param container Container to iterate over. begin() and end() methods /// must be available. /// \param converter Converter to apply to elements of the container. /// \return BackMap on success or std::nullopt on failure. In particular, if /// any exception is thrown from internal containers or converter returns an /// error, std::nullopt is returned. template [[nodiscard]] static auto Make(TContainer const* const container, Converter const& converter) noexcept -> std::unique_ptr { if (container == nullptr or converter == nullptr) { return nullptr; } auto const size = std::distance(container->begin(), container->end()); try { auto back_map = std::make_unique(); back_map->keys_.reserve(size); back_map->mapping_.reserve(size); for (auto const& value : *container) { std::optional key = BackMap::Convert(converter, value); if (not key.has_value()) { return nullptr; } if (auto inserted = back_map->keys_.insert(*std::move(key)); inserted.second) { // References and pointers to data stored in the container // are only invalidated by erasing that element, even when // the corresponding iterator is invalidated: // https://en.cppreference.com/w/cpp/container/unordered_set#:~:text=invalidated%20by%20erasing%20that%20element // According to ^, this approach is safe: back_map->mapping_.insert_or_assign(&*inserted.first, &value); } } return back_map; } catch (...) { return nullptr; } } template requires(std::is_invocable_v) [[nodiscard]] static auto Make(TContainer const* const container, TConverter const& converter) noexcept -> std::unique_ptr { using TResult = std::invoke_result_t; return Make(container, converter); } /// \brief Obtain all available keys. [[nodiscard]] auto GetKeys() const noexcept -> std::unordered_set const& { return keys_; } /// \brief Obtain a pointer to the value corresponding to the given key. /// Copy free. /// \return If the key is known, a pointer to the value is returned, /// otherwise std::nullopt. [[nodiscard]] auto GetReference(TKey const& key) const noexcept -> std::optional> { if (auto it = keys_.find(key); it != keys_.end()) { return mapping_.at(&*it); } return std::nullopt; } /// \brief Obtain the set of values corresponding to given keys. If a key /// isn't known to the container, it is ignored. Perform deep copy of /// referenced values. [[nodiscard]] auto GetValues(std::unordered_set const& keys) const noexcept -> std::unordered_set { std::unordered_set result; result.reserve(keys.size()); for (auto const& key : keys) { if (auto value = GetReference(key)) { result.emplace(*value.value()); } } return result; } /// \brief Obtain the set of values corresponding to given keys. If a key /// isn't known to the container, it is ignored. Copy free. [[nodiscard]] auto GetReferences(std::unordered_set const& keys) const noexcept -> std::unordered_set> { std::unordered_set> result; result.reserve(keys.size()); for (auto const& key : keys) { if (auto value = GetReference(key)) { result.emplace(*std::move(value)); } } return result; } using Iterable = std::unordered_map, gsl::not_null>; /// \brief Obtain an iterable key-value set where values correspond to the /// given keys. If a key isn't known to the container, it is ignored. Copy /// free. [[nodiscard]] auto IterateReferences( std::unordered_set const* keys) const noexcept -> Iterable { Iterable result; result.reserve(keys->size()); for (auto const& key : *keys) { if (auto value = GetReference(key)) { result.insert_or_assign(&key, value.value()); } } return result; } private: std::unordered_set keys_; std::unordered_map> mapping_; template [[nodiscard]] static auto Convert(Converter const& converter, TValue const& value) noexcept -> std::optional { if constexpr (not std::is_same_v) { TResult converted = std::invoke(converter, value); if (not static_cast(converted)) { return std::nullopt; } return *std::move(converted); } else { return std::invoke(converter, value); } } }; #endif // INCLUDED_SRC_UTILS_CPP_BACK_MAP_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/concepts.hpp000066400000000000000000000042641516554100600246650ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP #define INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP #include #include #include #include #include #include #include template concept ContainsString = requires { typename T::value_type; } and std::same_as; template concept HasSize = requires(T const c) { std::same_as; }; template concept InputIterableContainer = requires(T const c) { std::input_iterator; std::input_iterator; }; template concept OutputIterableContainer = requires(T c) { InputIterableContainer; std::output_iterator; }; template concept InputIterableStringContainer = InputIterableContainer and ContainsString; // TODO(modernize): remove this once we require clang version >= 14.0.0 template concept ClockHasFromSys = requires(std::chrono::time_point const tp) { T::from_sys(tp); }; // TODO(modernize): remove this once we require clang version >= 14.0.0 template concept ClockHasFromTime = requires(std::time_t const t) { T::from_time_t(t); }; template concept StrMapConstForwardIterator = requires(T const c) { { std::remove_reference_t{(*c).first} } -> std::same_as; }; #endif // INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/expected.hpp000066400000000000000000000051751516554100600246520ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_EXPECTED_HPP #define INCLUDED_SRC_UTILS_CPP_EXPECTED_HPP #include #include // TODO(modernize): replace this by std::unexpected once we switched to C++23 template class unexpected { // NOLINT(readability-identifier-naming) public: explicit unexpected(E error) : error_{std::move(error)} {} [[nodiscard]] auto error() && -> E { return std::move(error_); } private: E error_; }; // TODO(modernize): replace this by std::expected once we switched to C++23 template class expected { // NOLINT(readability-identifier-naming) public: expected(T value) noexcept // NOLINT : value_{std::in_place_index<0>, std::move(value)} {} expected(unexpected unexpected) noexcept // NOLINT : value_{std::in_place_index<1>, std::move(unexpected).error()} {} // Check whether the object contains an expected value [[nodiscard]] auto has_value() const noexcept -> bool { return value_.index() == 0; } [[nodiscard]] operator bool() const noexcept { // NOLINT return has_value(); } // Return the expected value [[nodiscard]] auto value() const& -> T const& { return std::get<0>(value_); } [[nodiscard]] auto value() && -> T { return std::move(std::get<0>(value_)); } // Access the expected value [[nodiscard]] auto operator*() const& noexcept -> T const& { return *std::get_if<0>(&value_); } [[nodiscard]] auto operator*() && noexcept -> T { return std::move(*std::get_if<0>(&value_)); } [[nodiscard]] auto operator->() const noexcept -> T const* { return std::get_if<0>(&value_); } // Access the unexpected value [[nodiscard]] auto error() const& noexcept -> const E& { return *std::get_if<1>(&value_); } [[nodiscard]] auto error() && noexcept -> E { return std::move(*std::get_if<1>(&value_)); } private: std::variant value_; }; #endif // INCLUDED_SRC_UTILS_CPP_EXPECTED_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/file_locking.cpp000066400000000000000000000107731516554100600254710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/file_locking.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include // for errno #include // for strerror() #include #include #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/path.hpp" auto LockFile::Acquire(std::filesystem::path const& fspath, bool is_shared) noexcept -> std::optional { static std::mutex lock_mutex{}; try { // ensure thread-safety std::unique_lock lock{lock_mutex}; // get path to lock file auto lock_file = GetLockFilePath(fspath); if (not lock_file) { return std::nullopt; } // touch lock file if (not FileSystemManager::CreateFile(*lock_file)) { Logger::Log(LogLevel::Error, "LockFile: could not create file {}", lock_file->string()); return std::nullopt; } // get open file descriptor gsl::owner file_handle = std::fopen(lock_file->c_str(), "r"); if (file_handle == nullptr) { Logger::Log(LogLevel::Error, "LockFile: could not open descriptor for file {}", lock_file->string()); return std::nullopt; } // attach flock auto err = flock(fileno(file_handle), is_shared ? LOCK_SH : LOCK_EX); if (err != 0) { Logger::Log(LogLevel::Error, "LockFile: applying lock to file {} failed with:\n{}", lock_file->string(), strerror(errno)); fclose(file_handle); return std::nullopt; } // lock file has been acquired return LockFile(file_handle, *lock_file); } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "LockFile: acquiring file lock for path {} failed with:\n{}", fspath.string(), ex.what()); return std::nullopt; } } LockFile::~LockFile() noexcept { if (file_handle_ != nullptr) { // close open file descriptor fclose(file_handle_); file_handle_ = nullptr; } } auto LockFile::GetLockFilePath(std::filesystem::path const& fspath) noexcept -> std::optional { try { // bring to normal form auto filename = ToNormalPath(fspath); if (not filename.is_absolute()) { try { filename = std::filesystem::absolute(fspath); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Failed to determine absolute path for lock file " "name {}: {}", fspath.string(), e.what()); return std::nullopt; } } auto parent = filename.parent_path(); // create parent folder if (not FileSystemManager::CreateDirectory(parent)) { return std::nullopt; } // return lock file name return filename; } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "LockFile: defining lock file name for path {} failed with:\n{}", fspath.string(), ex.what()); return std::nullopt; } } LockFile::LockFile(LockFile&& other) noexcept : file_handle_{other.file_handle_}, lock_file_{std::move(other.lock_file_)} { other.file_handle_ = nullptr; } auto LockFile::operator=(LockFile&& other) noexcept -> LockFile& { file_handle_ = other.file_handle_; other.file_handle_ = nullptr; lock_file_ = std::move(other.lock_file_); return *this; } just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/file_locking.hpp000066400000000000000000000046001516554100600254660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_FILE_LOCKING_HPP #define INCLUDED_SRC_OTHER_TOOLS_FILE_LOCKING_HPP #include #include #include #include // std::move #include "gsl/gsl" /* \brief Thread- and process-safe file locking mechanism for paths. * User guarantees write access in the parent directory of the path given, as * the lock will be placed there and missing tree directories will be created. */ class LockFile { public: // no default ctor LockFile() = delete; /// \brief Destroy the LockFile object. /// It only closes the open file descriptor, but does not explicitly unlock. ~LockFile() noexcept; // no copies, only moves LockFile(LockFile const&) = delete; LockFile(LockFile&& other) noexcept; auto operator=(LockFile const&) = delete; auto operator=(LockFile&& other) noexcept -> LockFile&; /// \brief Tries to acquire a lock file with the given name. /// Missing directories will be created if write permission exists. /// Returns the lock file object on success, nullopt on failure. [[nodiscard]] static auto Acquire(std::filesystem::path const& fspath, bool is_shared) noexcept -> std::optional; private: gsl::owner file_handle_{nullptr}; std::filesystem::path lock_file_; /// \brief Private ctor. Instances are only created by Acquire method. explicit LockFile(gsl::owner file_handle, std::filesystem::path lock_file) noexcept : file_handle_{file_handle}, lock_file_{std::move(lock_file)} {}; [[nodiscard]] static auto GetLockFilePath( std::filesystem::path const& fspath) noexcept -> std::optional; }; #endif // FILE_LOCKING just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/gsl.hpp000066400000000000000000000022561516554100600236330ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_GSL_HPP #define INCLUDED_SRC_UTILS_CPP_GSL_HPP // implement EnsuresAudit/ExpectsAudit (from gsl-lite) only run in debug mode #ifdef NDEBUG // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ExpectsAudit(x) (void)0 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define EnsuresAudit(x) (void)0 #else #include "gsl/gsl" // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ExpectsAudit(x) Expects(x) // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define EnsuresAudit(x) Ensures(x) #endif #endif // INCLUDED_SRC_UTILS_CPP_GSL_HPP_ just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/hash_combine.hpp000066400000000000000000000022541516554100600254630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_HASH_COMBINE_HPP #define INCLUDED_SRC_UTILS_CPP_HASH_COMBINE_HPP #include #include #include "gsl/gsl" // Taken from Boost, as hash_combine did not yet make it to STL. // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0814r0.pdf template inline auto hash_combine(gsl::not_null const& seed, T const& v) -> void { *seed ^= std::hash{}(v) + 0x9e3779b9 + (*seed << 6) + (*seed >> 2); // NOLINT } #endif // INCLUDED_SRC_UTILS_CPP_HASH_COMBINE_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/hex_string.hpp000066400000000000000000000035601516554100600252170ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP #define INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP #include #include #include #include #include #include #include [[nodiscard]] static inline auto IsHexString(std::string const& s) noexcept -> bool { return std::all_of( s.begin(), s.end(), [](unsigned char c) { return std::isxdigit(c); }); } [[nodiscard]] static inline auto ToHexString(std::string const& bytes) -> std::string { std::ostringstream ss{}; ss << std::hex << std::setfill('0'); for (auto const& b : bytes) { ss << std::setw(2) << static_cast(static_cast(b)); } return ss.str(); } [[nodiscard]] static inline auto FromHexString(std::string const& hexstring) -> std::optional { try { static constexpr std::size_t kHexBase = 16; std::stringstream ss{}; for (std::size_t i = 0; i < hexstring.length(); i += 2) { unsigned char c = std::stoul(hexstring.substr(i, 2), nullptr, kHexBase); ss << c; } return ss.str(); } catch (...) { return std::nullopt; } } #endif // INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/in_place_visitor.hpp000066400000000000000000000020141516554100600263670ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_IN_PLACE_VISITOR_HPP #define INCLUDED_SRC_UTILS_CPP_IN_PLACE_VISITOR_HPP #include template struct InPlaceVisitor final : TInvocable... { explicit constexpr InPlaceVisitor(TInvocable&&... f) : TInvocable{std::forward(f)}... {} using TInvocable::operator()...; }; #endif // INCLUDED_SRC_UTILS_CPP_IN_PLACE_VISITOR_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/incremental_reader.cpp000066400000000000000000000141561516554100600266660ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/incremental_reader.hpp" #include #include #include #include "fmt/core.h" #include "src/utils/cpp/in_place_visitor.hpp" namespace { void DisposeFile(gsl::owner file) noexcept { if (file == nullptr) { return; } std::fclose(file); } } // namespace auto IncrementalReader::FromFile(std::size_t chunk_size, std::filesystem::path const& path) noexcept -> expected { if (chunk_size == 0) { return unexpected{ "IncrementalReader: the chunk size cannot be 0"}; } try { // Ensure this is a file: if (not std::filesystem::is_regular_file(path)) { return unexpected{fmt::format( "IncrementalReader: not a file :\n {} ", path.string())}; } // Open file for reading: static constexpr std::string_view kReadBinary = "rb"; auto file = std::shared_ptr{ std::fopen(path.c_str(), kReadBinary.data()), ::DisposeFile}; if (file == nullptr) { return unexpected{ fmt::format("IncrementalReader: failed to open the file:\n{}", path.string())}; } std::size_t const content_size = std::filesystem::file_size(path); return IncrementalReader{chunk_size, content_size, std::move(file), /*buffer=*/std::string(chunk_size, '\0')}; } catch (std::exception const& e) { return unexpected{fmt::format( "IncrementalReader: While processing {}\ngot an exception: {}", path.string(), e.what())}; } catch (...) { return unexpected{fmt::format( "IncrementalReader: While processing {}\ngot an unknown exception", path.string())}; } } auto IncrementalReader::FromMemory( std::size_t chunk_size, gsl::not_null const& data) noexcept -> expected { if (chunk_size == 0) { return unexpected{ "IncrementalReader: the chunk size cannot be 0"}; } try { // Reading from memory doesn't require a buffer. The resulting chunks // point at the content_ directly. return IncrementalReader{chunk_size, /*content_size=*/data->size(), data, /*buffer=*/std::string{}}; } catch (...) { return unexpected{ "IncrementalReader: Got an unknown exception during creation from " "a string"}; } } auto IncrementalReader::ReadChunk(std::size_t offset) const noexcept -> expected { using Result = expected; InPlaceVisitor const visitor{ [this, offset](FileSource const& file) -> Result { return ReadFromFile(file, offset); }, [this, offset](MemorySource const& data) -> Result { return ReadFromMemory(data, offset); }, }; try { return std::visit(visitor, content_); } catch (std::exception const& e) { return unexpected{fmt::format( "IncrementalReader: ReadChunk got an exception:\n{}", e.what())}; } catch (...) { return unexpected{ "IncrementalReader: ReadChunk got an unknown exception"}; } } auto IncrementalReader::ReadFromFile(FileSource const& file, std::size_t offset) const -> expected { if (file == nullptr) { return unexpected{ "IncrementalReader: ReadFromFile: got corrupted file"}; } if (std::fseek(file.get(), gsl::narrow(offset), SEEK_SET) != 0) { return unexpected{ "IncrementalReader: ReadFromFile: failed to set offset"}; } std::size_t read = 0; while (std::feof(file.get()) == 0 and std::ferror(file.get()) == 0 and read < buffer_.size()) { read += std::fread( &buffer_[read], sizeof(char), buffer_.size() - read, file.get()); } if (std::ferror(file.get()) != 0) { return unexpected{ fmt::format("IncrementalReader: ReadFromFile: ferror {}", std::ferror(file.get()))}; } return std::string_view{buffer_.data(), read}; } auto IncrementalReader::ReadFromMemory(MemorySource const& data, std::size_t offset) const -> expected { if (data->empty()) { // NOLINTNEXTLINE(bugprone-string-constructor,-warnings-as-errors) return std::string_view{data->data(), 0}; } std::size_t const read = std::min(chunk_size_, data->size() - offset); return std::string_view{&data->at(offset), read}; } IncrementalReader::Iterator::Iterator( gsl::not_null const& owner, std::size_t offset) noexcept : owner_{owner}, offset_{offset} {} auto IncrementalReader::Iterator::operator*() const noexcept -> expected { return owner_->ReadChunk(offset_); } auto IncrementalReader::Iterator::operator++() noexcept -> IncrementalReader::Iterator& { offset_ += owner_->chunk_size_; if (offset_ >= owner_->content_size_) { offset_ = owner_->GetEndOffset(); } return *this; } just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/incremental_reader.hpp000066400000000000000000000127141516554100600266710ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_INCREMENTAL_READER_HPP #define INCLUDED_SRC_UTILS_CPP_INCREMENTAL_READER_HPP #include #include #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/utils/cpp/expected.hpp" /// \brief Read data from source incrementally chunk by chunk. /// - Ensures that chunks are exactly the specified size if EOF is not reached. /// - Ensures no allocations happen while reading. Uses pre-allocated buffer and /// utilizes std::string_view. /// - Guarantees to return at least one chunk for an empty source. class IncrementalReader final { class Iterator final { public: using value_type = expected; using pointer = value_type*; using reference = value_type&; using difference_type = std::ptrdiff_t; using iterator_category = std::forward_iterator_tag; explicit Iterator(gsl::not_null const& owner, std::size_t offset) noexcept; auto operator*() const noexcept -> value_type; auto operator++() noexcept -> Iterator&; [[nodiscard]] friend auto operator==(Iterator const& lhs, Iterator const& rhs) noexcept -> bool { return lhs.owner_.get() == rhs.owner_.get() and lhs.offset_ == rhs.offset_; } [[nodiscard]] friend auto operator!=(Iterator const& lhs, Iterator const& rhs) noexcept -> bool { return not(lhs == rhs); } private: // Store by pointer to allow copies: gsl::not_null owner_; std::size_t offset_; }; public: /// \brief Create IncrementalReader that uses the given file as the source /// of data. /// \param chunk_size Size of chunk, must be greater than 0. /// \param path File to read. /// \return Configured reader on success or an error message on failure. [[nodiscard]] static auto FromFile( std::size_t chunk_size, std::filesystem::path const& path) noexcept -> expected; /// \brief Create IncrementalReader that uses the given string as the source /// of data. /// \param chunk_size Size of chunk, must be greater than 0. /// \param data String to read. /// \return Configured reader on success or an error message on failure. [[nodiscard]] static auto FromMemory( std::size_t chunk_size, gsl::not_null const& data) noexcept -> expected; [[nodiscard]] auto GetContentSize() const noexcept -> std::size_t { return content_size_; } /// \brief Create an iterator corresponding to the given offset. If the /// offset exceeds the maximum content size, it is adjusted. [[nodiscard]] auto make_iterator(std::size_t offset) const noexcept -> Iterator { return Iterator{this, std::min(offset, GetEndOffset())}; } [[nodiscard]] auto begin() const& noexcept -> Iterator { return make_iterator(/*offset=*/0); } [[nodiscard]] auto end() const& noexcept -> Iterator { return make_iterator(GetEndOffset()); } private: using FileSource = std::shared_ptr; using MemorySource = gsl::not_null; using ContentSource = std::variant; std::size_t chunk_size_; std::size_t content_size_; ContentSource content_; mutable std::string buffer_; explicit IncrementalReader(std::size_t chunk_size, std::size_t content_size, ContentSource content, std::string buffer) noexcept : chunk_size_{chunk_size}, content_size_{content_size}, content_{std::move(content)}, buffer_{std::move(buffer)} {} [[nodiscard]] auto ReadChunk(std::size_t offset) const noexcept -> expected; [[nodiscard]] auto ReadFromFile(FileSource const& file, std::size_t offset) const -> expected; [[nodiscard]] auto ReadFromMemory(MemorySource const& data, std::size_t offset) const -> expected; /// \brief Obtain offset corresponding to the end of content. The content /// size is shifted by 1 character to properly handle empty sources. [[nodiscard]] auto GetEndOffset() const noexcept -> std::size_t { return content_size_ + 1; } }; #endif // INCLUDED_SRC_UTILS_CPP_INCREMENTAL_READER_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/json.hpp000066400000000000000000000200331516554100600240100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_JSON_HPP #define INCLUDED_SRC_UTILS_CPP_JSON_HPP #include #include #include #include #include #include #include #include #include #include #include "fmt/core.h" #include "fmt/ostream.h" #include "nlohmann/json.hpp" #include "src/utils/cpp/gsl.hpp" template auto ExtractValueAs( nlohmann::json const& j, std::string const& key, std::function&& logger = [](std::string const& /*unused*/) -> void { }) noexcept -> std::optional { try { auto it = j.find(key); if (it == j.end()) { logger("key " + key + " cannot be found in JSON object"); return std::nullopt; } return it.value().template get(); } catch (std::exception& e) { logger(e.what()); return std::nullopt; } } namespace detail { [[nodiscard]] static inline auto IndentListsOnlyUntilDepth( nlohmann::json const& json, std::string const& indent, std::size_t until, std::size_t depth) -> std::string { using iterator = std::ostream_iterator; if (json.is_object()) { std::size_t i{}; std::ostringstream oss{}; oss << '{' << std::endl; for (auto const& [key, value] : json.items()) { std::fill_n(iterator{oss}, depth + 1, indent); oss << nlohmann::json(key).dump() << ": " << IndentListsOnlyUntilDepth(value, indent, until, depth + 1) << (++i == json.size() ? "" : ",") << std::endl; } std::fill_n(iterator{oss}, depth, indent); oss << '}'; EnsuresAudit(nlohmann::json::parse(oss.str()) == json); return oss.str(); } if (json.is_array() and depth < until) { std::size_t i{}; std::ostringstream oss{}; oss << '[' << std::endl; for (auto const& value : json) { std::fill_n(iterator{oss}, depth + 1, indent); oss << IndentListsOnlyUntilDepth(value, indent, until, depth + 1) << (++i == json.size() ? "" : ",") << std::endl; } std::fill_n(iterator{oss}, depth, indent); oss << ']'; EnsuresAudit(nlohmann::json::parse(oss.str()) == json); return oss.str(); } return json.dump(); } [[nodiscard]] static inline auto IndentOnlyUntilDepth( nlohmann::json const& json, std::string const& indent, std::size_t until, std::size_t depth, std::optional path, std::unordered_map const& depths) -> std::string { using iterator = std::ostream_iterator; if (path and depths.find(*path) != depths.end()) { until = depths.find(*path)->second; } if (json.is_object() and depth < until) { std::size_t i{}; std::ostringstream oss{}; oss << '{' << std::endl; for (auto const& [key, value] : json.items()) { std::fill_n(iterator{oss}, depth + 1, indent); oss << nlohmann::json(key).dump() << ": " << IndentOnlyUntilDepth( value, indent, until, depth + 1, path ? std::optional(*path + "/" + key) : std::nullopt, depths) << (++i == json.size() ? "" : ",") << std::endl; } std::fill_n(iterator{oss}, depth, indent); oss << '}'; EnsuresAudit(nlohmann::json::parse(oss.str()) == json); return oss.str(); } if (json.is_array() and depth < until) { std::size_t i{}; std::ostringstream oss{}; oss << '[' << std::endl; for (auto const& value : json) { std::fill_n(iterator{oss}, depth + 1, indent); oss << IndentOnlyUntilDepth( value, indent, until, depth + 1, std::nullopt, depths) << (++i == json.size() ? "" : ",") << std::endl; } std::fill_n(iterator{oss}, depth, indent); oss << ']'; EnsuresAudit(nlohmann::json::parse(oss.str()) == json); return oss.str(); } return json.dump(); } } // namespace detail /// \brief Dump json with indent. Indent lists only until specified depth. [[nodiscard]] static inline auto IndentListsOnlyUntilDepth( nlohmann::json const& json, std::size_t indent, std::size_t until_depth = 0) -> std::string { return detail::IndentListsOnlyUntilDepth( json, std::string(indent, ' '), until_depth, 0); } // \brief Dump json with indent. Indent until the given list; for initial // pure object-paths, alternative depths can be specified [[nodiscard]] static inline auto IndentOnlyUntilDepth( nlohmann::json const& json, std::size_t indent, std::size_t until_depth, std::unordered_map const& depths) -> std::string { return detail::IndentOnlyUntilDepth( json, std::string(indent, ' '), until_depth, 0, "", depths); } // \brief Dump json, replacing subexpressions at the given depths by "*". [[nodiscard]] static inline auto TruncateJson(nlohmann::json const& json, std::size_t depth) -> std::string { if (depth == 0) { return "*"; } if (json.is_object()) { std::size_t i{}; std::ostringstream oss{}; oss << '{'; for (auto const& [key, value] : json.items()) { oss << nlohmann::json(key).dump() << ":" << TruncateJson(value, depth - 1) << (++i == json.size() ? "" : ","); } oss << '}'; return oss.str(); } if (json.is_array()) { std::size_t i{}; std::ostringstream oss{}; oss << '['; for (auto const& value : json) { oss << TruncateJson(value, depth - 1) << (++i == json.size() ? "" : ","); } oss << ']'; return oss.str(); } return json.dump(); } [[nodiscard]] static inline auto AbbreviateJson(nlohmann::json const& json, std::size_t len) -> std::string { auto json_string = json.dump(); if (json_string.size() <= len) { return json_string; } std::size_t i = 1; auto old_abbrev = TruncateJson(json, i); auto new_abbrev = old_abbrev; while (new_abbrev.size() <= len) { old_abbrev = new_abbrev; ++i; new_abbrev = TruncateJson(json, i); } return old_abbrev; } // \brief For a json object, return an object that is obtained from the original // one by dropping all top-level keys where the value is null. [[nodiscard]] static inline auto PruneJson(nlohmann::json const& json) -> nlohmann::json { if (not json.is_object()) { return json; } auto result = nlohmann::json::object(); for (auto const& el : json.items()) { if (not el.value().is_null()) { result[el.key()] = el.value(); } } return result; } #if defined(FMT_VERSION) and FMT_VERSION >= 100000 // Use nlohmann::basic_json::operator<<() for formatting via libfmt. // This explicit template specialization seems to be required starting with // libfmt 10.x. template <> struct fmt::formatter> : ostream_formatter {}; #endif #endif // INCLUDED_SRC_UTILS_CPP_JSON_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/path.hpp000066400000000000000000000052341516554100600240010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_PATH_HPP #define INCLUDED_SRC_UTILS_CPP_PATH_HPP #include #include [[nodiscard]] static inline auto ToNormalPath( std::filesystem::path const& p) noexcept -> std::filesystem::path { auto n = p.lexically_normal(); if (not n.has_filename()) { n = n.parent_path(); } if (n.empty()) { return std::filesystem::path{"."}; } return n; } /// \brief Perform a non-upwards condition check on the given path. /// A path is non-upwards if it is relative and it never references any other /// path on a higher level in the directory tree than itself. [[nodiscard]] static inline auto PathIsNonUpwards( std::filesystem::path const& path) noexcept -> bool { if (path.is_absolute()) { return false; } // check non-upwards condition return *path.lexically_normal().begin() != ".."; } /// \brief Perform a confined condition check on a path with respect to /// another path. A path is confined wrt another if it is relative and results /// in a non-upwards path when applied from the directory of the other path. /// This models the situation when a symlink is being resolved inside a tree. /// NOTE: No explicit check is done whether applied_to is actually a relative /// path, as this is implicit by the non-upwardness condition. [[nodiscard]] static inline auto PathIsConfined( std::filesystem::path const& path, std::filesystem::path const& applied_to) noexcept -> bool { if (path.is_absolute()) { return false; } // check confined upwards condition; this call also handles the case when // applied_to is an absolute path return PathIsNonUpwards(applied_to.parent_path() / path); } /// \brief Predicate if a given string is a valid filename. [[nodiscard]] static inline auto IsValidFileName(const std::string& s) -> bool { if (s.find_first_of("/\0") != std::string::npos) { return false; } if (s.empty() or s == "." or s == "..") { return false; } return true; } #endif // INCLUDED_SRC_UTILS_CPP_PATH_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/path_hash.hpp000066400000000000000000000030061516554100600247770ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_PATH_HASH_HPP #define INCLUDED_SRC_UTILS_CPP_PATH_HASH_HPP // IWYU pragma: always_keep // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define GLIBCXX_11_4 20230528 // gcc/DATESTAMP of version 11.4 #if (defined(__GLIBCXX__) and _GLIBCXX_RELEASE < 12 and \ !(_GLIBCXX_RELEASE == 11 and __GLIBCXX__ >= GLIBCXX_11_4)) or \ (defined(_LIBCPP_VERSION) and _LIBCPP_VERSION < 16000) #include #include // std::hash is missing for // - GNU's libstdc++ < 11.4 // - LLVM's libcxx < 16 (see https://reviews.llvm.org/D125394) namespace std { template <> struct hash { [[nodiscard]] auto operator()( std::filesystem::path const& ct) const noexcept -> std::size_t { return std::filesystem::hash_value(ct); } }; } // namespace std #endif #endif // INCLUDED_SRC_UTILS_CPP_PATH_HASH_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/path_rebase.hpp000066400000000000000000000026011516554100600253150ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_PATH_REBASE_HPP #define INCLUDED_SRC_UTILS_CPP_PATH_REBASE_HPP #include #include #include [[nodiscard]] static inline auto RebasePathStringRelativeTo( const std::string& base, const std::string& path) -> std::string { return std::filesystem::path(path).lexically_relative(base).string(); } [[nodiscard]] static inline auto RebasePathStringsRelativeTo( const std::string& base, const std::vector& paths) -> std::vector { std::vector result{}; result.reserve(paths.size()); for (auto const& path : paths) { result.emplace_back(RebasePathStringRelativeTo(base, path)); } return result; } #endif // INCLUDED_SRC_UTILS_CPP_PATH_REBASE_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/prefix.hpp000066400000000000000000000021731516554100600243410ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_PREFIX_HPP #define INCLUDED_SRC_UTILS_CPP_PREFIX_HPP #include #include [[nodiscard]] static inline auto PrefixLines(const std::string& text, const std::string& prefix = " ") -> std::string { std::stringstream out{}; std::istringstream in{text}; for (std::string line{}; std::getline(in, line);) { out << prefix << line << "\n"; } return out.str(); } #endif // INCLUDED_SRC_UTILS_CPP_PREFIX_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/tmp_dir.cpp000066400000000000000000000057661516554100600245100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/tmp_dir.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include #include #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" auto TmpDir::Create(std::filesystem::path const& prefix) noexcept -> Ptr { return CreateImpl(/*parent=*/nullptr, prefix); } auto TmpDir::CreateNestedDirectory(TmpDir::Ptr const& parent) noexcept -> TmpDir::Ptr { if (parent == nullptr) { return nullptr; } return CreateImpl(parent, parent->GetPath()); } auto TmpDir::CreateFile(TmpDir::Ptr const& parent, std::string const& file_name) noexcept -> TmpFile::Ptr { // Create a new tmp directory to guarantee uniqueness: auto temp_dir = CreateNestedDirectory(parent); if (temp_dir == nullptr) { return nullptr; } try { auto file_path = std::filesystem::weakly_canonical(temp_dir->GetPath() / file_name); if (not FileSystemManager::CreateFile(file_path)) { return nullptr; } return std::shared_ptr{ new TmpFile(std::move(temp_dir), std::move(file_path))}; } catch (...) { return nullptr; } } auto TmpDir::CreateImpl(TmpDir::Ptr parent, std::filesystem::path const& path) noexcept -> Ptr { static constexpr std::string_view kDirTemplate = "tmp.XXXXXX"; // make sure prefix folder exists if (not FileSystemManager::CreateDirectory(path)) { Logger::Log(LogLevel::Error, "TmpDir: could not create prefix directory {}", path.string()); return nullptr; } std::string file_path; try { file_path = std::filesystem::weakly_canonical(path / kDirTemplate); // Create a temporary directory: if (mkdtemp(file_path.data()) == nullptr) { return nullptr; } return std::shared_ptr( new TmpDir(std::move(parent), file_path)); } catch (...) { if (not file_path.empty()) { rmdir(file_path.c_str()); } } return nullptr; } TmpDir::~TmpDir() noexcept { // try to remove the tmp dir and all its content std::ignore = FileSystemManager::RemoveDirectory(tmp_dir_); } just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/tmp_dir.hpp000066400000000000000000000061151516554100600245020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_OTHER_TOOLS_TMP_DIR_HPP #define INCLUDED_SRC_OTHER_TOOLS_TMP_DIR_HPP #include #include #include #include class TmpFile; class TmpDir final { public: using Ptr = std::shared_ptr; TmpDir(TmpDir const&) = delete; auto operator=(TmpDir const&) -> TmpDir& = delete; TmpDir(TmpDir&& other) = delete; auto operator=(TmpDir&&) -> TmpDir& = delete; ~TmpDir() noexcept; [[nodiscard]] auto GetPath() const& noexcept -> std::filesystem::path const& { return tmp_dir_; } /// \brief Creates a completely unique directory in a given prefix path. [[nodiscard]] static auto Create( std::filesystem::path const& prefix) noexcept -> Ptr; /// \brief Create a new nested temporary directory. This nested directory /// remains valid even if the parent directory goes out of scope. [[nodiscard]] static auto CreateNestedDirectory( TmpDir::Ptr const& parent) noexcept -> Ptr; /// \brief Create a new unique temporary file. To guarantee uniqueness, /// every file gets created in a new temporary directory. This file remains /// valid even if the parent directory goes out of scope. [[nodiscard]] static auto CreateFile( TmpDir::Ptr const& parent, std::string const& file_name = "file") noexcept -> std::shared_ptr; private: explicit TmpDir(TmpDir::Ptr parent, std::filesystem::path path) noexcept : parent_{std::move(parent)}, tmp_dir_{std::move(path)} {} [[nodiscard]] static auto CreateImpl( TmpDir::Ptr parent, std::filesystem::path const& path) noexcept -> Ptr; TmpDir::Ptr parent_; std::filesystem::path tmp_dir_; }; class TmpFile final { friend class TmpDir; public: using Ptr = std::shared_ptr; TmpFile(TmpFile const&) = delete; auto operator=(TmpFile const&) -> TmpFile& = delete; TmpFile(TmpFile&& other) = delete; auto operator=(TmpFile&&) -> TmpFile& = delete; ~TmpFile() noexcept = default; [[nodiscard]] auto GetPath() const& noexcept -> std::filesystem::path const& { return file_path_; } private: explicit TmpFile(TmpDir::Ptr parent, std::filesystem::path file_path) noexcept : parent_{std::move(parent)}, file_path_{std::move(file_path)} {} TmpDir::Ptr parent_; std::filesystem::path file_path_; }; #endif // INCLUDED_SRC_OTHER_TOOLS_TMP_DIR_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/type_safe_arithmetic.hpp000066400000000000000000000074361516554100600272430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP #define INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP #include #include "gsl/gsl" /// \struct TypeSafeArithmeticTag /// \brief Abstract tag defining types and limits for custom arithmetic types. /// Usage example: /// struct my_type_tag : TypeSafeArithmeticTag {}; /// using my_type_t = TypeSafeArithmetic; template ::lowest(), T kMax = std::numeric_limits::max(), T kSmallest = std::numeric_limits::min()> struct TypeSafeArithmeticTag { static_assert(std::is_arithmetic_v, "T must be an arithmetic type (integer or floating-point)"); using value_t = T; using reference_t = T&; using const_reference_t = T const&; using pointer_t = T*; using const_pointer_t = T const*; static constexpr value_t kMaxValue = kMax; static constexpr value_t kMinValue = kMin; static constexpr value_t kSmallestValue = kSmallest; }; /// \class TypeSafeArithmetic /// \brief Abstract class for defining custom arithmetic types. /// \tparam TAG The actual \ref TypeSafeArithmeticTag template class TypeSafeArithmetic { typename TAG::value_t value_{}; public: using tag_t = TAG; using value_t = typename tag_t::value_t; using reference_t = typename tag_t::reference_t; using const_reference_t = typename tag_t::const_reference_t; using pointer_t = typename tag_t::pointer_t; using const_pointer_t = typename tag_t::const_pointer_t; static constexpr value_t kMaxValue = tag_t::kMaxValue; static constexpr value_t kMinValue = tag_t::kMinValue; static constexpr value_t kSmallestValue = tag_t::kSmallestValue; constexpr TypeSafeArithmetic() = default; // NOLINTNEXTLINE constexpr /*explicit*/ TypeSafeArithmetic(value_t value) { set(value); } TypeSafeArithmetic(TypeSafeArithmetic const&) = default; TypeSafeArithmetic(TypeSafeArithmetic&&) noexcept = default; auto operator=(TypeSafeArithmetic const&) -> TypeSafeArithmetic& = default; auto operator=(TypeSafeArithmetic&&) noexcept -> TypeSafeArithmetic& = default; ~TypeSafeArithmetic() = default; auto operator=(value_t value) -> TypeSafeArithmetic& { set(value); return *this; } // NOLINTNEXTLINE constexpr /*explicit*/ operator value_t() const { return value_; } constexpr auto get() const -> value_t { return value_; } constexpr void set(value_t value) { Expects(value >= kMinValue and value <= kMaxValue and "value output of range"); value_ = value; } auto pointer() const -> const_pointer_t { return &value_; } }; template auto operator+=(TypeSafeArithmetic& lhs, TypeSafeArithmetic rhs) -> TypeSafeArithmetic& { lhs.set(lhs.get() + rhs.get()); return lhs; } template auto operator++(TypeSafeArithmetic& a, int) -> TypeSafeArithmetic { auto r = a; a += TypeSafeArithmetic{1}; return r; } #endif // INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP just-buildsystem-justbuild-b1fb5fa/src/utils/cpp/vector.hpp000066400000000000000000000021031516554100600243370ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_UTILS_CPP_VECTOR_HPP #define INCLUDED_SRC_UTILS_CPP_VECTOR_HPP // small library to manipulate vectors #include // IWYU pragma: keep #include // sort the passed vector and remove repeated entries template void sort_and_deduplicate(std::vector* x) { std::sort(x->begin(), x->end()); auto it = std::unique(x->begin(), x->end()); x->erase(it, x->end()); } #endif // INCLUDED_SRC_UTILS_CPP_VECTOR_HPP just-buildsystem-justbuild-b1fb5fa/test/000077500000000000000000000000001516554100600205765ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/TARGETS000066400000000000000000000146001516554100600216330ustar00rootroot00000000000000{ "tested just": { "type": ["@", "rules", "CC", "install-with-deps"] , "skip-debug-stage": ["yes"] , "targets": [["@", "src", "", "just"]] } , "tool-under-test": { "type": "install" , "dirs": [["tested just", "."]] , "files": {"bin/tool-under-test": ["@", "src", "", "just"]} } , "tested just-mr": { "type": ["@", "rules", "CC", "install-with-deps"] , "skip-debug-stage": ["yes"] , "targets": [["@", "src", "", "just-mr"]] } , "mr-tool-under-test": { "type": "install" , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "dirs": { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": [["tested just-mr", "."]] } , "files": { "bin/mr-tool-under-test": { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": ["@", "src", "", "bin/just-mr.py"] , "else": ["@", "src", "", "just-mr"] } } } , "catch-main": { "type": ["@", "rules", "CC", "library"] , "name": ["catch-main"] , "srcs": ["main.cpp"] , "deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "git_context"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "file_chunker"] , ["utils", "log_config"] , ["utils", "test_env"] ] , "stage": ["test"] } , "unit tests with compatibility (unconfigured)": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_COMPATIBLE_REMOTE"] , "stage": [ { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": "compatible" , "else": "native" } ] , "deps": [["./", "buildtool", "TESTS"]] } , "unit tests with compatibility, compatible": { "type": "configure" , "arguments_config": ["TEST_ENV"] , "tainted": ["test"] , "target": "unit tests with compatibility (unconfigured)" , "config": { "type": "`" , "$1": { "TEST_COMPATIBLE_REMOTE": true , "TEST_ENV": { "type": "," , "$1": { "type": "map_union" , "$1": [ { "type": "var" , "name": "TEST_ENV" , "default": {"type": "empty_map"} } , {"type": "'", "$1": {"COMPATIBLE": "YES"}} ] } } } } } , "unit tests with compatibility, native": { "type": "configure" , "arguments_config": ["TEST_ENV"] , "tainted": ["test"] , "target": "unit tests with compatibility (unconfigured)" , "config": { "type": "`" , "$1": { "TEST_COMPATIBLE_REMOTE": false , "TEST_ENV": { "type": "," , "$1": { "type": "map_union" , "$1": { "type": "foreach_map" , "var_key": "name" , "var_val": "value" , "range": { "type": "var" , "name": "TEST_ENV" , "default": {"type": "empty_map"} } , "body": { "type": "case" , "expr": {"type": "var", "name": "name"} , "case": {"COMPATIBLE": {"type": "empty_map"}} , "default": { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "var", "name": "value"} } } } } } } } } , "unit tests with compatibility": { "type": ["@", "rules", "test", "suite"] , "stage": ["using-remote"] , "deps": [ "unit tests with compatibility, compatible" , "unit tests with compatibility, native" ] } , "TESTS": { "type": ["@", "rules", "test", "matrix"] , "arguments_config": ["DROP_LARGE_TESTS", "DROP_END_TO_END_TESTS"] , "deps": { "type": "`" , "$1": [ "unit tests with compatibility" , ["./", "utils", "TESTS"] , ["./", "other_tools", "TESTS"] , { "type": ",@" , "$1": { "type": "if" , "cond": {"type": "var", "name": "DROP_END_TO_END_TESTS"} , "then": [] , "else": [["./", "end-to-end", "TESTS"]] } } , { "type": ",@" , "$1": { "type": "if" , "cond": { "type": "or" , "$1": [ {"type": "var", "name": "DROP_END_TO_END_TESTS"} , {"type": "var", "name": "DROP_LARGE_TESTS"} ] } , "then": [] , "else": ["bootstrap-test"] } } ] } } , "ALL": { "type": "configure" , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "tainted": ["test"] , "target": "TESTS" , "config": { "type": "let*" , "bindings": [ ["OS", {"type": "var", "name": "OS", "default": "linux"}] , ["ARCH", {"type": "var", "name": "ARCH", "default": "x86_64"}] , [ "HOST_ARCH" , { "type": "var" , "name": "HOST_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] , [ "TARGET_ARCH" , { "type": "var" , "name": "TARGET_ARCH" , "default": {"type": "var", "name": "ARCH"} } ] ] , "body": {"type": "env", "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"]} } } , "UNIT_TESTS": { "type": "configure" , "tainted": ["test"] , "target": "ALL" , "config": {"type": "'", "$1": {"DROP_END_TO_END_TESTS": true}} } , "bootstrap-test": { "type": "configure" , "tainted": ["test"] , "target": ["./", "bootstrap", "TESTS"] , "arguments_config": ["TIMEOUT_SCALE"] , "config": { "type": "singleton_map" , "key": "TIMEOUT_SCALE" , "value": { "type": "*" , "$1": [40, {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0}] } } } , "test-deps-headers": { "type": ["@", "rules", "CC", "install-with-deps"] , "hdrs-only": ["yes"] , "targets": [["@", "catch2", "", "catch2"]] } , "just-ext-hdrs (unconfigured)": { "type": "install" , "dirs": [[["@", "src", "", "just-ext-hdrs"], "."], ["test-deps-headers", "."]] } , "just-ext-hdrs": { "type": "configure" , "arguments_config": ["OS", "ARCH"] , "target": "just-ext-hdrs (unconfigured)" , "config": { "type": "let*" , "bindings": [ ["OS", {"type": "var", "name": "OS", "default": "linux"}] , ["ARCH", {"type": "var", "name": "ARCH", "default": "x86_64"}] ] , "body": {"type": "env", "vars": ["OS", "ARCH"]} } } } just-buildsystem-justbuild-b1fb5fa/test/bootstrap/000077500000000000000000000000001516554100600226135ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/bootstrap/RULES000066400000000000000000000142271516554100600234360ustar00rootroot00000000000000{ "stage-deps": { "doc": [ "Set up depending libraries in a way they could occur in a distribution" , "" , "Runfiles and \"compile-deps\" of \"libs\" are staged to \"include\"," , "artifacts and \"link-deps\" to \"lib\", in a flat way." , "Artifacts of \"bins\" are staged to \"bin\"." , "The source files underlying \"protos\" are staged to \"proto\"." , "The artifacts of \"deps\" are directly added to the stage." , "This is only for testing bootstrapping against preinstalled dependencies." ] , "tainted": ["test"] , "target_fields": ["libs", "bins", "protos", "deps"] , "string_fields": ["prefix"] , "anonymous": { "proto-srcs": { "target": "protos" , "provider": "proto" , "rule_map": {"library": "stage-proto", "service library": "stage-proto"} } } , "expression": { "type": "let*" , "bindings": [ [ "prefix" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "prefix"} } ] , [ "runfiles" , { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": [ { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "libs"} , "body": { "type": "DEP_RUNFILES" , "dep": {"type": "var", "name": "dep"} } } , { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "libs"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "dep"} , "provider": "compile-deps" , "default": {"type": "empty_map"} } } ] } } ] , [ "artifacts" , { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": [ { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "libs"} , "body": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "dep"} } } , { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "libs"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "dep"} , "provider": "link-deps" , "default": {"type": "empty_map"} } } ] } } ] , [ "bins" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "bins"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "dep"}} } } ] , [ "protos" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "proto-srcs"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "dep"}} } } ] , [ "deps" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "dep"}} } } ] , [ "stage" , { "type": "map_union" , "$1": [ { "type": "to_subdir" , "subdir": "include" , "$1": {"type": "var", "name": "runfiles"} } , { "type": "to_subdir" , "subdir": "lib" , "flat": true , "$1": {"type": "var", "name": "artifacts"} } , { "type": "to_subdir" , "subdir": "bin" , "$1": {"type": "var", "name": "bins"} } , { "type": "to_subdir" , "subdir": "include" , "$1": {"type": "var", "name": "protos"} } , {"type": "var", "name": "deps"} ] } ] , [ "stage" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "prefix"} , "$1": {"type": "var", "name": "stage"} } ] ] , "body": { "type": "RESULT" , "runfiles": {"type": "var", "name": "stage"} , "artifacts": {"type": "var", "name": "stage"} } } } , "stage-proto": { "string_fields": ["name", "stage"] , "target_fields": ["srcs", "deps"] , "expression": { "type": "let*" , "bindings": [ [ "srcs" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": "srcs"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "src"}} } } ] , [ "srcs" , { "type": "to_subdir" , "subdir": { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } , "$1": {"type": "var", "name": "srcs"} } ] , [ "transitive srcs" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "src" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "src"}} } } ] , [ "all srcs" , { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "srcs"} , {"type": "var", "name": "transitive srcs"} ] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "all srcs"}} } } } just-buildsystem-justbuild-b1fb5fa/test/bootstrap/TARGETS000066400000000000000000000057331516554100600236570ustar00rootroot00000000000000{ "bundled-test": { "type": ["@", "rules", "shell/test", "script"] , "arguments_config": ["SUFFIX"] , "name": [ "bootstrap-test-bundled" , {"type": "var", "name": "SUFFIX", "default": ""} ] , "test": ["test-bootstrap.sh"] , "deps": [ "prune-config.py" , ["@", "just-distfiles", "", "distdir"] , ["@", "src", "", "bootstrap-src"] ] } , "bundled-test-debug": { "type": "configure" , "tainted": ["test"] , "target": "bundled-test" , "config": {"type": "'", "$1": {"SUFFIX": "-debug", "TEST_ENV": {"DEBUG": "YES"}}} } , "bundled-test-gnu": { "type": "configure" , "tainted": ["test"] , "target": "bundled-test" , "config": { "type": "'" , "$1": { "SUFFIX": "-gnu" , "TEST_ENV": { "JUST_BUILD_CONF": "{\"TOOLCHAIN_CONFIG\":{\"FAMILY\": \"gnu\"}, \"CC\": \"gcc\", \"CXX\": \"g++\"}" } } } } , "distro-bootstrap-deps": { "type": "install" , "dirs": [["just_cc_deps", "LOCALBASE"], ["just_other_deps", "LOCALBASE"]] , "tainted": ["test"] } , "just_cc_deps": { "type": ["@", "rules", "CC", "install-with-deps"] , "targets": [ ["@", "json", "", "json"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "cli11", "", "cli11"] , ["@", "ssl", "", "crypto"] , ["@", "src", "", "libgit2"] , ["@", "protoc", "", "protoc"] , ["@", "protoc", "", "libprotobuf"] , ["@", "grpc", "", "grpc++"] , ["@", "grpc", "", "grpc"] , ["@", "grpc", "", "grpc_cpp_plugin"] , ["@", "src", "", "libarchive"] ] } , "just_other_deps": { "type": ["bootstrap", "stage-deps"] , "protos": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] , ["@", "googleapis", "", "google_bytestream_proto"] , ["@", "googleapis", "", "google_api_httpbody_proto"] , ["@", "googleapis", "", "google_api_expr_v1alpha1_checked_proto"] , ["@", "googleapis", "", "google_api_expr_v1alpha1_syntax_proto"] ] } , "staged-sources": { "type": "install" , "tainted": ["test"] , "dirs": [[["@", "src", "", "bootstrap-src"], "srcs/just"]] } , "pkgconfig-test": { "type": ["@", "rules", "shell/test", "script"] , "name": ["bootstrap-test-pkgconfig"] , "test": ["test-bootstrap-pkgconfig.sh"] , "deps": ["distro-bootstrap-deps", "staged-sources"] } , "mixed-test": { "type": ["@", "rules", "shell/test", "script"] , "name": ["bootstrap-test-mixed"] , "test": ["test-mixed-bootstrap.sh"] , "deps": [ "distro-bootstrap-deps" , "staged-sources" , ["@", "just-distfiles", "", "distdir"] ] } , "symlink-test": { "type": ["@", "rules", "shell/test", "script"] , "name": ["bootstrap-test-symlink"] , "test": ["test-symlink-bootstrap.sh"] , "deps": ["distro-bootstrap-deps", "staged-sources"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["bootstrap"] , "deps": [ "bundled-test" , "bundled-test-debug" , "bundled-test-gnu" , "mixed-test" , "pkgconfig-test" , "symlink-test" ] } } just-buildsystem-justbuild-b1fb5fa/test/bootstrap/prune-config.py000077500000000000000000000020301516554100600255570ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import sys src: str = sys.argv[1] dest: str = sys.argv[2] empty_dir: str = sys.argv[3] with open(src) as f: repos = json.load(f) for repo in repos["repositories"]: desc = repos["repositories"][repo] if desc.get("bootstrap", {}).get("drop"): desc["repository"] = {"type": "file", "path": empty_dir} with open(dest, "w") as f: json.dump(repos, f, indent=2) just-buildsystem-justbuild-b1fb5fa/test/bootstrap/test-bootstrap-pkgconfig.sh000077500000000000000000000036641516554100600301220ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu # Set paths export PATH=/bin:/usr/bin:$PATH readonly LOCALBASE=`pwd`/LOCALBASE readonly WRKSRC=`pwd`/srcs/just readonly WRKDIR=${TMPDIR}/work-just-bootstrap mkdir -p ${WRKDIR} readonly TESTDIR=${TMPDIR}/work-test mkdir -p ${TESTDIR} readonly TEST_BUILD_ROOT=${TMPDIR}/.just mkdir -p ${TEST_BUILD_ROOT} readonly TEST_OUT_DIR=${TMPDIR}/work-test-out mkdir -p ${TEST_OUT_DIR} # bootstrap command env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY": "clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} 2>&1 # Do some sanity checks with the binary JUST=${WRKDIR}/out/bin/just echo Bootstrap finished. Obtained ${JUST} echo ${JUST} -h echo ${JUST} version echo touch ${TESTDIR}/ROOT cat > ${TESTDIR}/TARGETS <<'EOF' { "hello world": { "type": "generic" , "cmds": ["echo Hello World > out.txt"] , "outs": ["out.txt"] } , "": { "type": "generic" , "cmds": ["cat out.txt | tr a-z A-Z > final.txt"] , "outs": ["final.txt"] , "deps": ["hello world"] } } EOF echo ${JUST} install -o ${TEST_OUT_DIR} --workspace-root ${TESTDIR} --local-build-root ${TEST_BUILD_ROOT} '' '' ${JUST} install -o ${TEST_OUT_DIR} --workspace-root ${TESTDIR} --local-build-root ${TEST_BUILD_ROOT} '' '' grep HELLO ${TEST_OUT_DIR}/final.txt echo OK just-buildsystem-justbuild-b1fb5fa/test/bootstrap/test-bootstrap.sh000066400000000000000000000030611516554100600261410ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu export PATH=/bin:/usr/bin:$PATH readonly OUTDIR=${TEST_TMPDIR}/out readonly LBRDIR=${TEST_TMPDIR}/local-build-root readonly EMPTY=${TEST_TMPDIR}/empty readonly PRUNED_CONFIG=${TEST_TMPDIR}/pruned_config mkdir -p "${OUTDIR}" mkdir -p "${LBRDIR}" mkdir -p "${EMPTY}" BOOTSTRAP_BIN=./bin/bootstrap.py echo echo Bootstrap echo python3 ${BOOTSTRAP_BIN} . "${OUTDIR}"/boot distdir export JUST=$(realpath "${OUTDIR}"/boot/out/bin/just) echo echo Testing if we reached the fixed point echo ./prune-config.py etc/repos.json ${PRUNED_CONFIG} ${EMPTY} cat ${PRUNED_CONFIG} echo readonly CONF=$(./bin/just-mr.py -C ${PRUNED_CONFIG} --distdir=distdir --local-build-root="${LBRDIR}" setup just) : ${JUST_BUILD_CONF:="{}"} ${JUST} install -C ${CONF} -D "${JUST_BUILD_CONF}" -o "${OUTDIR}"/final-out --local-build-root="${LBRDIR}" sha256sum "${OUTDIR}"/boot/out/bin/just "${OUTDIR}"/final-out/bin/just cmp "${OUTDIR}"/boot/out/bin/just "${OUTDIR}"/final-out/bin/just just-buildsystem-justbuild-b1fb5fa/test/bootstrap/test-mixed-bootstrap.sh000077500000000000000000000044241516554100600272540ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e # Set paths export PATH=/bin:/usr/bin:$PATH readonly ORIG_LOCALBASE=`pwd`/LOCALBASE readonly LOCALBASE=${TMPDIR}/new-localbase readonly WRKSRC=`pwd`/srcs/just readonly WRKDIR=${TMPDIR}/work-just-bootstrap mkdir -p ${WRKDIR} readonly TESTDIR=${TMPDIR}/work-test mkdir -p ${TESTDIR} readonly TEST_BUILD_ROOT=${TMPDIR}/.just mkdir -p ${TEST_BUILD_ROOT} readonly TEST_OUT_DIR=${TMPDIR}/work-test-out mkdir -p ${TEST_OUT_DIR} readonly DISTDIR=${TMPDIR}/distdir mkdir -p "${DISTDIR}" # Set up local base, leaving out some dependencies cp -r "${ORIG_LOCALBASE}" "${LOCALBASE}" # - gsl rm -rf "${LOCALBASE}/include/gsl" cp distdir/v4.0.0.tar.gz "${DISTDIR}" # - fmt rm -rf "${LOCALBASE}/include/fmt*" rm -rf "${LOCALBASE}/lib/libfmt*" cp distdir/fmt-11.2.0.zip "${DISTDIR}" # bootstrap command env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ NON_LOCAL_DEPS='["com_github_microsoft_gsl", "fmt"]' \ JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY":"clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} ${DISTDIR} 2>&1 # Do some sanity checks with the binary JUST=${WRKDIR}/out/bin/just echo Bootstrap finished. Obtained ${JUST} echo ${JUST} -h echo ${JUST} version echo touch ${TESTDIR}/ROOT cat > ${TESTDIR}/TARGETS <<'EOF' { "hello world": { "type": "generic" , "cmds": ["echo Hello World > out.txt"] , "outs": ["out.txt"] } , "": { "type": "generic" , "cmds": ["cat out.txt | tr a-z A-Z > final.txt"] , "outs": ["final.txt"] , "deps": ["hello world"] } } EOF ${JUST} install -o ${TEST_OUT_DIR} --workspace-root ${TESTDIR} --local-build-root ${TEST_BUILD_ROOT} '' '' grep HELLO ${TEST_OUT_DIR}/final.txt echo OK just-buildsystem-justbuild-b1fb5fa/test/bootstrap/test-symlink-bootstrap.sh000077500000000000000000000046011516554100600276310ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e # Set paths export PATH=/bin:/usr/bin:$PATH readonly ORIG_LOCALBASE=`pwd`/LOCALBASE readonly UNRELATED_FILE=${TMPDIR}/unrealated_file touch ${UNRELATED_FILE} readonly LOCALBASE=${TMPDIR}/new-localbase readonly WRKSRC=`pwd`/srcs/just readonly WRKDIR=${TMPDIR}/work-just-bootstrap mkdir -p ${WRKDIR} readonly TESTDIR=${TMPDIR}/work-test mkdir -p ${TESTDIR} readonly TEST_BUILD_ROOT=${TMPDIR}/.just mkdir -p ${TEST_BUILD_ROOT} readonly TEST_OUT_DIR=${TMPDIR}/work-test-out mkdir -p ${TEST_OUT_DIR} # Set up local base, adding some symbolic links # bin/protoc is a symbolic link cp -r "${ORIG_LOCALBASE}" "${LOCALBASE}" mv "${LOCALBASE}/bin/protoc" "${LOCALBASE}/bin/protoc.actual_binary" ln -s "${LOCALBASE}/bin/protoc.actual_binary" "${LOCALBASE}/bin/protoc" # directories contain unrelated symlinks ln -s ${UNRELATED_FILE} ${LOCALBASE}/bin/foo ln -s ${UNRELATED_FILE} ${LOCALBASE}/include/foo.h ln -s ${UNRELATED_FILE} ${LOCALBASE}/lib/libfoo.so # bootstrap command env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY":"clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} 2>&1 # Do some sanity checks with the binary JUST=${WRKDIR}/out/bin/just echo Bootstrap finished. Obtained ${JUST} echo ${JUST} -h echo ${JUST} version echo touch ${TESTDIR}/ROOT cat > ${TESTDIR}/TARGETS <<'EOF' { "hello world": { "type": "generic" , "cmds": ["echo Hello World > out.txt"] , "outs": ["out.txt"] } , "": { "type": "generic" , "cmds": ["cat out.txt | tr a-z A-Z > final.txt"] , "outs": ["final.txt"] , "deps": ["hello world"] } } EOF ${JUST} install -o ${TEST_OUT_DIR} --workspace-root ${TESTDIR} --local-build-root ${TEST_BUILD_ROOT} '' '' grep HELLO ${TEST_OUT_DIR}/final.txt echo OK just-buildsystem-justbuild-b1fb5fa/test/buildtool/000077500000000000000000000000001516554100600225735ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/TARGETS000066400000000000000000000011421516554100600236250ustar00rootroot00000000000000{ "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["buildtool"] , "deps": [ ["./", "build_engine", "TESTS"] , ["./", "common", "TESTS"] , ["./", "crypto", "TESTS"] , ["./", "execution_api", "TESTS"] , ["./", "execution_engine", "TESTS"] , ["./", "file_system", "TESTS"] , ["./", "graph_traverser", "TESTS"] , ["./", "logging", "TESTS"] , ["./", "main", "TESTS"] , ["./", "multithreading", "TESTS"] , ["./", "serve_api", "TESTS"] , ["./", "storage", "TESTS"] , ["./", "system", "TESTS"] , ["./", "tree_structure", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/000077500000000000000000000000001516554100600252175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/TARGETS000066400000000000000000000003301516554100600262470ustar00rootroot00000000000000{ "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["build_engine"] , "deps": [ ["./", "base_maps", "TESTS"] , ["./", "expression", "TESTS"] , ["./", "target_map", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/000077500000000000000000000000001516554100600271515ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/TARGETS000066400000000000000000000150171516554100600302110ustar00rootroot00000000000000{ "test_repo": { "type": ["@", "rules", "CC", "library"] , "name": ["test_repo"] , "hdrs": ["test_repo.hpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["utils", "shell_quoting"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "entity_name": { "type": ["@", "rules", "CC/test", "test"] , "name": ["entity_name"] , "srcs": ["entity_name.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "directory_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["directory_map"] , "srcs": ["directory_map.test.cpp"] , "data": ["test_data_src", "test_data"] , "private-deps": [ "test_repo" , ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/build_engine/base_maps", "directory_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "module_name"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "json_file_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["json_file_map"] , "srcs": ["json_file_map.test.cpp"] , "data": ["test_data_json", "test_data"] , "private-deps": [ "test_repo" , ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/build_engine/base_maps", "json_file_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "module_name"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "source_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["source_map"] , "srcs": ["source_map.test.cpp"] , "data": ["test_data", "test_data_src"] , "private-deps": [ "test_repo" , ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/base_maps", "directory_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["@", "src", "src/buildtool/build_engine/base_maps", "source_map"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "expression_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["expression_map"] , "srcs": ["expression_map.test.cpp"] , "data": ["test_data_expr", "test_data"] , "private-deps": [ "test_repo" , ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["@", "src", "src/buildtool/build_engine/base_maps", "expression_map"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "rule_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["rule_map"] , "srcs": ["rule_map.test.cpp"] , "data": ["test_data_rule", "test_data"] , "private-deps": [ "test_repo" , ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["@", "src", "src/buildtool/build_engine/base_maps", "expression_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "rule_map"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "test_data_src": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data_src/file", "data_src/foo/bar/file"] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "test_data_json": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data_json/bad.json", "data_json/foo.json"] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "test_data_expr": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data_expr/EXPRESSIONS", "data_expr/readers/EXPRESSIONS"] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "test_data_rule": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data_rule/RULES", "data_rule/composers/EXPRESSIONS"] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data/test_repo.bundle"] , "stage": ["test", "buildtool", "build_engine", "base_maps"] } , "data/test_repo.bundle": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["create_maps_test_git_bundle.sh"] , "outs": ["data/test_repo.bundle"] , "cmds": ["sh create_maps_test_git_bundle.sh"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["base_maps"] , "deps": [ "directory_map" , "entity_name" , "expression_map" , "json_file_map" , "rule_map" , "source_map" ] } } create_maps_test_git_bundle.sh000066400000000000000000000224631516554100600351530ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e if ! command -v git >/dev/null; then echo "git is required to run this script." exit 1 fi # --- # Structure of test_repo: # --- # # | # +--src <--kSrcTreeId (6d57ba31821f69286e280334e4fd5f9dbd141721) # | +--file # | +--foo # | | +--link <--kSrcLinkId (2995a4d0e74917fd3e1383c577d0fc301fff1b04) # | | +--bar # | | | +--file # | # +--expr <--kExprTreeId (4946bd21d0a5b3e0c82d6944f3d47adaf1bb66f7) # | +--EXRESSIONS # | +--readers # | | +--EXPRESSIONS # | # +--rule <--kRuleTreeId (c6dd902c9d4e7afa8b20eb04e58503e63ecab84d) # | +--RULES # | +--composers # | | +--EXPRESSIONS # | # +--json <--kJsonTreeId (6982563dfc4dcdd1362792dbbc9d8243968d1ec9) # | +--data_json # | | +--bad.json # | | +--foo.json # | # --- # Repository root folder: test_repo # Bundle name: test_repo.bundle # Metadata: test_repo.json # # create the root folder mkdir -p test_repo cd test_repo # create the repo git init > /dev/null 2>&1 git checkout -q -b master git config user.name "Nobody" git config user.email "nobody@example.org" # create src tree mkdir -p src/foo/bar touch src/file touch src/foo/bar/file ln -s dummy src/foo/link # create src commit git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "src" > /dev/null # create expr tree mkdir -p expr/readers cat < expr/EXPRESSIONS { "test_expression_literal": { "expression": "foo" }, "test_read_vars": { "vars": ["FOO"], "expression": { "type": "var", "name": "FOO" } }, "test_call_import": { "vars": ["FOO"], "imports": { "read_foo": [ "readers", "real_foo_reader" ] }, "expression": { "type": "CALL_EXPRESSION", "name": "read_foo" } }, "test_overwrite_import": { "vars": ["FOO"], "imports": { "read_foo": [ "readers", "proxy_foo_reader" ] }, "expression": { "type": "CALL_EXPRESSION", "name": "read_foo" } }, "test_missing_vars": { "expression": { "type": "var", "name": "FOO" } }, "test_missing_imports": { "expression": { "type": "CALL_EXPRESSION", "name": "read_foo" } }, "test_malformed_function": "not_an_object", "test_malformed_expression": { "missing_expression": {} }, "test_malformed_vars": { "vars": "not_a_list", "expression": { "type": "empty_map" } }, "test_malformed_imports": { "imports": "not_an_object", "expression": { "type": "empty_map" } } } EOF cat < expr/readers/EXPRESSIONS { "proxy_foo_reader": { "vars": [ "FOO" ], "imports": { "read_foo": "real_foo_reader" }, "expression": { "type": "CALL_EXPRESSION", "name": "read_foo" } }, "real_foo_reader": { "vars": [ "FOO" ], "expression": { "type": "var", "name": "FOO" } } } EOF # create expr commit git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "expr" > /dev/null # create rule tree mkdir -p rule/composers cat < rule/RULES { "test_empty_rule": { "expression": { "type": "RESULT" } }, "test_rule_fields": { "string_fields": ["foo"], "target_fields": ["bar"], "config_fields": ["baz"], "expression": { "type": "RESULT" } }, "test_config_transitions_target_via_field": { "target_fields": ["target"], "config_transitions": { "target": [{ "type": "empty_map" }] }, "expression": { "type": "RESULT" } }, "test_config_transitions_target_via_implicit": { "implicit": { "target": [ ["module", "name"] ] }, "config_transitions": { "target": [{ "type": "empty_map" }] }, "expression": { "type": "RESULT" } }, "test_config_transitions_canonicalness": { "target_fields": ["foo", "bar"], "string_fields": ["quux", "corge"], "config_fields": ["grault", "garply"], "implicit": { "baz": [ ["module", "name"] ], "qux": [ ["module", "name"] ] }, "config_transitions": { "bar": [{ "type": "singleton_map", "key": "exists", "value": true }], "qux": [{ "type": "singleton_map", "key": "defined", "value": true }] }, "expression": { "type": "RESULT" } }, "test_call_import": { "config_vars": ["FOO"], "imports": { "compose_foo": [ "composers", "foo_composer" ] }, "expression": { "type": "CALL_EXPRESSION", "name": "compose_foo" } }, "test_string_kw_conflict": { "string_fields": ["foo", "type", "bar"], "expression": { "type": "RESULT" } }, "test_target_kw_conflict": { "target_fields": ["foo", "arguments_config", "bar"], "expression": { "type": "RESULT" } }, "test_config_kw_conflict": { "config_fields": ["foo", "type", "bar"], "expression": { "type": "RESULT" } }, "test_implicit_kw_conflict": { "implicit": { "foo": [], "arguments_config": [], "bar": [] }, "expression": { "type": "RESULT" } }, "test_string_target_conflict": { "string_fields": ["foo", "bar"], "target_fields": ["bar", "baz"], "expression": { "type": "RESULT" } }, "test_target_config_conflict": { "target_fields": ["foo", "bar"], "config_fields": ["bar", "baz"], "expression": { "type": "RESULT" } }, "test_config_implicit_conflict": { "config_fields": ["foo", "bar"], "implicit": { "bar": [ ["module", "name"] ], "baz": [ ["module", "name"] ] }, "expression": { "type": "RESULT" } }, "test_unknown_config_transitions_target": { "config_transitions": { "missing": [{ "type": "empty_map" }] }, "expression": { "type": "RESULT" } }, "test_missing_config_vars": { "imports": { "compose_foo": [ "composers", "foo_composer" ] }, "expression": { "type": "CALL_EXPRESSION", "name": "compose_foo" } }, "test_missing_imports": { "expression": { "type": "CALL_EXPRESSION", "name": "compose_foo" } }, "test_malformed_rule": "not_an_object", "test_malformed_rule_expression": { "missing_expression": { "type": "RESULT" } }, "test_malformed_target_fields": { "target_fields": "not_a_list", "expression": { "type": "RESULT" } }, "test_malformed_string_fields": { "string_fields": "not_a_list", "expression": { "type": "RESULT" } }, "test_malformed_config_fields": { "config_fields": "not_a_list", "expression": { "type": "RESULT" } }, "test_malformed_implicit": { "implicit": "not_an_object", "expression": { "type": "RESULT" } }, "test_malformed_implicit_entry": { "implicit": { "target": "not_a_list" }, "expression": { "type": "RESULT" } }, "test_malformed_implicit_entity_name": { "implicit": { "target": [ ["module_without_name"] ] }, "expression": { "type": "RESULT" } }, "test_malformed_config_vars": { "config_vars": "not_a_list", "expression": { "type": "RESULT" } }, "test_malformed_config_transitions": { "config_transitions": "not_an_object", "expression": { "type": "RESULT" } }, "test_malformed_imports": { "imports": "not_an_object", "expression": { "type": "RESULT" } } } EOF cat < rule/composers/EXPRESSIONS { "foo_composer": { "vars": [ "FOO" ], "expression": { "type": "map_union", "\$1": [{ "type": "singleton_map", "key": "type", "value": "RESULT" }, { "type": "singleton_map", "key": "artifacts", "value": { "type": "singleton_map", "key": "foo", "value": { "type": "var", "name": "FOO" } } } ] } } } EOF # create rule commit git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "rule" > /dev/null # create json tree mkdir -p json/data_json cat < json/data_json/bad.json This is not JSON EOF cat < json/data_json/foo.json { "foo": "bar" } EOF # create json commit git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "json" > /dev/null # create the git bundle git bundle create ../data/test_repo.bundle HEAD master just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_expr/000077500000000000000000000000001516554100600311205ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_expr/EXPRESSIONS000066400000000000000000000016511516554100600326700ustar00rootroot00000000000000{ "test_expression_literal": {"expression": "foo"} , "test_read_vars": {"vars": ["FOO"], "expression": {"type": "var", "name": "FOO"}} , "test_call_import": { "vars": ["FOO"] , "imports": {"read_foo": ["readers", "real_foo_reader"]} , "expression": {"type": "CALL_EXPRESSION", "name": "read_foo"} } , "test_overwrite_import": { "vars": ["FOO"] , "imports": {"read_foo": ["readers", "proxy_foo_reader"]} , "expression": {"type": "CALL_EXPRESSION", "name": "read_foo"} } , "test_missing_vars": {"expression": {"type": "var", "name": "FOO"}} , "test_missing_imports": {"expression": {"type": "CALL_EXPRESSION", "name": "read_foo"}} , "test_malformed_function": "not_an_object" , "test_malformed_expression": {"missing_expression": {}} , "test_malformed_vars": {"vars": "not_a_list", "expression": {"type": "empty_map"}} , "test_malformed_imports": {"imports": "not_an_object", "expression": {"type": "empty_map"}} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_expr/readers/000077500000000000000000000000001516554100600325455ustar00rootroot00000000000000EXPRESSIONS000066400000000000000000000003701516554100600342330ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_expr/readers{ "proxy_foo_reader": { "vars": ["FOO"] , "imports": {"read_foo": "real_foo_reader"} , "expression": {"type": "CALL_EXPRESSION", "name": "read_foo"} } , "real_foo_reader": {"vars": ["FOO"], "expression": {"type": "var", "name": "FOO"}} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_json/000077500000000000000000000000001516554100600311135ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_json/bad.json000066400000000000000000000000211516554100600325250ustar00rootroot00000000000000This is not JSON just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_json/foo.json000066400000000000000000000000231516554100600325640ustar00rootroot00000000000000{ "foo": "bar" } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_rule/000077500000000000000000000000001516554100600311115ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_rule/RULES000066400000000000000000000072761516554100600317420ustar00rootroot00000000000000{ "test_empty_rule": {"expression": {"type": "RESULT"}} , "test_rule_fields": { "string_fields": ["foo"] , "target_fields": ["bar"] , "config_fields": ["baz"] , "expression": {"type": "RESULT"} } , "test_config_transitions_target_via_field": { "target_fields": ["target"] , "config_transitions": {"target": [{"type": "empty_map"}]} , "expression": {"type": "RESULT"} } , "test_config_transitions_target_via_implicit": { "implicit": {"target": [["module", "name"]]} , "config_transitions": {"target": [{"type": "empty_map"}]} , "expression": {"type": "RESULT"} } , "test_config_transitions_canonicalness": { "target_fields": ["foo", "bar"] , "string_fields": ["quux", "corge"] , "config_fields": ["grault", "garply"] , "implicit": {"baz": [["module", "name"]], "qux": [["module", "name"]]} , "config_transitions": { "bar": [{"type": "singleton_map", "key": "exists", "value": true}] , "qux": [{"type": "singleton_map", "key": "defined", "value": true}] } , "expression": {"type": "RESULT"} } , "test_call_import": { "config_vars": ["FOO"] , "imports": {"compose_foo": ["composers", "foo_composer"]} , "expression": {"type": "CALL_EXPRESSION", "name": "compose_foo"} } , "test_string_kw_conflict": {"string_fields": ["foo", "type", "bar"], "expression": {"type": "RESULT"}} , "test_target_kw_conflict": { "target_fields": ["foo", "arguments_config", "bar"] , "expression": {"type": "RESULT"} } , "test_config_kw_conflict": {"config_fields": ["foo", "type", "bar"], "expression": {"type": "RESULT"}} , "test_implicit_kw_conflict": { "implicit": {"foo": [], "arguments_config": [], "bar": []} , "expression": {"type": "RESULT"} } , "test_string_target_conflict": { "string_fields": ["foo", "bar"] , "target_fields": ["bar", "baz"] , "expression": {"type": "RESULT"} } , "test_target_config_conflict": { "target_fields": ["foo", "bar"] , "config_fields": ["bar", "baz"] , "expression": {"type": "RESULT"} } , "test_config_implicit_conflict": { "config_fields": ["foo", "bar"] , "implicit": {"bar": [["module", "name"]], "baz": [["module", "name"]]} , "expression": {"type": "RESULT"} } , "test_unknown_config_transitions_target": { "config_transitions": {"missing": [{"type": "empty_map"}]} , "expression": {"type": "RESULT"} } , "test_missing_config_vars": { "imports": {"compose_foo": ["composers", "foo_composer"]} , "expression": {"type": "CALL_EXPRESSION", "name": "compose_foo"} } , "test_missing_imports": {"expression": {"type": "CALL_EXPRESSION", "name": "compose_foo"}} , "test_malformed_rule": "not_an_object" , "test_malformed_rule_expression": {"missing_expression": {"type": "RESULT"}} , "test_malformed_target_fields": {"target_fields": "not_a_list", "expression": {"type": "RESULT"}} , "test_malformed_string_fields": {"string_fields": "not_a_list", "expression": {"type": "RESULT"}} , "test_malformed_config_fields": {"config_fields": "not_a_list", "expression": {"type": "RESULT"}} , "test_malformed_implicit": {"implicit": "not_an_object", "expression": {"type": "RESULT"}} , "test_malformed_implicit_entry": {"implicit": {"target": "not_a_list"}, "expression": {"type": "RESULT"}} , "test_malformed_implicit_entity_name": { "implicit": {"target": [["module_without_name"]]} , "expression": {"type": "RESULT"} } , "test_malformed_implicit_entity_name_2": {"implicit": {"target": [[]]}, "expression": {"type": "RESULT"}} , "test_malformed_config_vars": {"config_vars": "not_a_list", "expression": {"type": "RESULT"}} , "test_malformed_config_transitions": {"config_transitions": "not_an_object", "expression": {"type": "RESULT"}} , "test_malformed_imports": {"imports": "not_an_object", "expression": {"type": "RESULT"}} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_rule/composers/000077500000000000000000000000001516554100600331235ustar00rootroot00000000000000EXPRESSIONS000066400000000000000000000006171516554100600346150ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_rule/composers{ "foo_composer": { "vars": ["FOO"] , "expression": { "type": "map_union" , "$1": [ {"type": "singleton_map", "key": "type", "value": "RESULT"} , { "type": "singleton_map" , "key": "artifacts" , "value": { "type": "singleton_map" , "key": "foo" , "value": {"type": "var", "name": "FOO"} } } ] } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_src/000077500000000000000000000000001516554100600307315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_src/file000066400000000000000000000000001516554100600315610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_src/foo/000077500000000000000000000000001516554100600315145ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_src/foo/bar/000077500000000000000000000000001516554100600322605ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/data_src/foo/bar/file000066400000000000000000000000001516554100600331100ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/directory_map.test.cpp000066400000000000000000000073671516554100600335110ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include #include #include #include // std::move #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/build_engine/base_maps/test_repo.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { using namespace BuildMaps::Base; // NOLINT auto SetupConfig(StorageConfig const* storage_config, bool use_git) -> RepositoryConfig { auto root = FileRoot{kBasePath / "data_src"}; if (use_git) { auto repo_path = CreateTestRepo(); REQUIRE(repo_path); REQUIRE(storage_config); auto git_root = FileRoot::FromGit(storage_config, *repo_path, kSrcTreeId); REQUIRE(git_root); root = std::move(*git_root); } RepositoryConfig repo_config{}; repo_config.SetInfo("", RepositoryConfig::RepositoryInfo{root}); return repo_config; } auto ReadDirectory(ModuleName const& id, DirectoryEntriesMap::Consumer value_checker, StorageConfig const* storage_config, bool use_git = false) -> bool { auto repo_config = SetupConfig(storage_config, use_git); auto data_direntries = CreateDirectoryEntriesMap(&repo_config); bool success{true}; { TaskSystem ts; data_direntries.ConsumeAfterKeysReady( &ts, {id}, std::move(value_checker), [&success](std::string const& /*unused*/, bool /*unused*/) { success = false; }); } return success; } } // namespace TEST_CASE("simple usage") { bool as_expected{false}; auto name = ModuleName{"", "."}; auto consumer = [&as_expected](auto values) { if (values[0]->ContainsBlob("file") and not values[0]->ContainsBlob("does_not_exist")) { as_expected = true; }; }; SECTION("via file") { CHECK(ReadDirectory(name, consumer, nullptr, /*use_git=*/false)); CHECK(as_expected); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadDirectory( name, consumer, &storage_config.Get(), /*use_git=*/true)); CHECK(as_expected); } } TEST_CASE("missing directory") { bool as_expected{false}; auto name = ModuleName{"", "does_not_exist"}; auto consumer = [&as_expected](auto values) { if (values[0]->Empty()) { as_expected = true; } }; SECTION("via file") { CHECK(ReadDirectory(name, consumer, nullptr, /*use_git=*/false)); CHECK(as_expected); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadDirectory( name, consumer, &storage_config.Get(), /*use_git=*/true)); CHECK(as_expected); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/entity_name.test.cpp000066400000000000000000000032141516554100600331470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" TEST_CASE("Normal module names") { using NT = BuildMaps::Base::NamedTarget; CHECK(NT::normal_module_name("foo/bar") == "foo/bar"); CHECK(NT::normal_module_name("foo/bar/") == "foo/bar"); CHECK(NT::normal_module_name("./foo/bar") == "foo/bar"); CHECK(NT::normal_module_name("/foo/bar") == "foo/bar"); CHECK(NT::normal_module_name("/foo/bar/.") == "foo/bar"); CHECK(NT::normal_module_name("/foo/bar/baz/..") == "foo/bar"); CHECK(NT::normal_module_name("foo/baz/../bar") == "foo/bar"); CHECK(NT::normal_module_name("../../../foo/bar") == "foo/bar"); CHECK(NT::normal_module_name("").empty()); CHECK(NT::normal_module_name(".").empty()); CHECK(NT::normal_module_name("./").empty()); CHECK(NT::normal_module_name("./.").empty()); CHECK(NT::normal_module_name("/").empty()); CHECK(NT::normal_module_name("/.").empty()); CHECK(NT::normal_module_name("..").empty()); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/expression_map.test.cpp000066400000000000000000000206341516554100600336740ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include #include #include #include // std::move #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/build_engine/base_maps/test_repo.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { using namespace BuildMaps::Base; // NOLINT auto SetupConfig(StorageConfig const* storage_config, bool use_git) -> RepositoryConfig { auto root = FileRoot{kBasePath / "data_expr"}; if (use_git) { auto repo_path = CreateTestRepo(); REQUIRE(repo_path); REQUIRE(storage_config); auto git_root = FileRoot::FromGit(storage_config, *repo_path, kExprTreeId); REQUIRE(git_root); root = std::move(*git_root); } RepositoryConfig repo_config{}; repo_config.SetInfo("", RepositoryConfig::RepositoryInfo{root}); return repo_config; } auto ReadExpressionFunction(EntityName const& id, ExpressionFunctionMap::Consumer value_checker, StorageConfig const* storage_config = nullptr, bool use_git = false) -> bool { auto repo_config = SetupConfig(storage_config, use_git); auto expr_file_map = CreateExpressionFileMap(&repo_config, 0); auto expr_func_map = CreateExpressionMap(&expr_file_map, &repo_config); bool success{true}; { TaskSystem ts; expr_func_map.ConsumeAfterKeysReady( &ts, {id}, std::move(value_checker), [&success](std::string const& /*unused*/, bool /*unused*/) { success = false; }); } return success; } } // namespace TEST_CASE("Simple expression object literal", "[expression_map]") { auto name = EntityName{"", ".", "test_expression_literal"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0])->Evaluate({}, {}); REQUIRE(expr); REQUIRE(expr->IsString()); CHECK(expr == Expression::FromJson(R"("foo")"_json)); }; SECTION("via file") { CHECK( ReadExpressionFunction(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadExpressionFunction( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Simple read of variable", "[expression_map]") { auto name = EntityName{"", ".", "test_read_vars"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0]) ->Evaluate(Configuration{Expression::FromJson( R"({"FOO": "bar"})"_json)}, {}); REQUIRE(expr); REQUIRE(expr->IsString()); CHECK(expr == Expression{std::string{"bar"}}); }; SECTION("via file") { CHECK( ReadExpressionFunction(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadExpressionFunction( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Simple call of imported expression", "[expression_map]") { auto name = EntityName{"", ".", "test_call_import"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0]) ->Evaluate(Configuration{Expression::FromJson( R"({"FOO": "bar"})"_json)}, {}); REQUIRE(expr); REQUIRE(expr->IsString()); CHECK(expr == Expression{std::string{"bar"}}); }; SECTION("via file") { CHECK( ReadExpressionFunction(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadExpressionFunction( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Overwrite import in nested expression", "[expression_map]") { auto name = EntityName{"", ".", "test_overwrite_import"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0]) ->Evaluate(Configuration{Expression::FromJson( R"({"FOO": "bar"})"_json)}, {}); REQUIRE(expr); REQUIRE(expr->IsString()); CHECK(expr == Expression{std::string{"bar"}}); }; SECTION("via file") { CHECK( ReadExpressionFunction(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadExpressionFunction( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Fail due to unkown ID", "[expression_map]") { auto name = EntityName{"", ".", "does_not_exist"}; auto consumer = [](auto /*values*/) { CHECK(false); // should never be called }; SECTION("via file") { CHECK_FALSE( ReadExpressionFunction(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK_FALSE(ReadExpressionFunction( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Fail due to missing vars", "[expression_map]") { CHECK( ReadExpressionFunction({"", ".", "test_missing_vars"}, [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0]) ->Evaluate(Configuration{Expression::FromJson( R"({"FOO": "bar"})"_json)}, {}); CHECK(expr == Expression::FromJson(R"(null)"_json)); })); } TEST_CASE("Fail due to missing imports", "[expression_map]") { CHECK(ReadExpressionFunction( {"", ".", "test_missing_imports"}, [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0]) ->Evaluate(Configuration{Expression::FromJson( R"({"FOO": "bar"})"_json)}, {}); CHECK_FALSE(expr); })); } TEST_CASE("Malformed function", "[expression_map]") { CHECK_FALSE(ReadExpressionFunction( {"", ".", "test_malformed_function"}, [](auto /*values*/) { CHECK(false); // should never be called })); } TEST_CASE("Malformed expression", "[expression_map]") { CHECK_FALSE(ReadExpressionFunction( {"", ".", "test_malformed_expression"}, [](auto /*values*/) { CHECK(false); // should never be called })); } TEST_CASE("Malformed vars", "[expression_map]") { CHECK_FALSE(ReadExpressionFunction( {"", ".", "test_malformed_vars"}, [](auto /*values*/) { CHECK(false); // should never be called })); } TEST_CASE("Malformed imports", "[expression_map]") { CHECK_FALSE(ReadExpressionFunction( {"", ".", "test_malformed_imports"}, [](auto /*values*/) { CHECK(false); // should never be called })); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/json_file_map.test.cpp000066400000000000000000000150011516554100600334350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/json_file_map.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/build_engine/base_maps/module_name.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/build_engine/base_maps/test_repo.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { using namespace BuildMaps::Base; // NOLINT auto SetupConfig(std::string target_file_name, StorageConfig const* storage_config, bool use_git) -> RepositoryConfig { auto root = FileRoot{kBasePath}; if (use_git) { auto repo_path = CreateTestRepo(); REQUIRE(repo_path); REQUIRE(storage_config); auto git_root = FileRoot::FromGit(storage_config, *repo_path, kJsonTreeId); REQUIRE(git_root); root = std::move(*git_root); } auto info = RepositoryConfig::RepositoryInfo{root}; info.target_file_name = std::move(target_file_name); RepositoryConfig repo_config{}; repo_config.SetInfo("", std::move(info)); return repo_config; } template auto ReadJsonFile(std::string const& target_file_name, ModuleName const& id, JsonFileMap::Consumer value_checker, StorageConfig const* storage_config, bool use_git = false, std::optional fail_func = std::nullopt) -> bool { auto repo_config = SetupConfig(target_file_name, storage_config, use_git); auto json_files = CreateJsonFileMap<&RepositoryConfig::WorkspaceRoot, &RepositoryConfig::TargetFileName, kMandatory>(&repo_config, 0); bool success{true}; { TaskSystem ts; json_files.ConsumeAfterKeysReady( &ts, {id}, std::move(value_checker), [&success](std::string const& /*unused*/, bool /*unused*/) { success = false; }, fail_func ? std::move(*fail_func) : [] {}); } return success; } } // namespace TEST_CASE("simple usage") { bool as_expected{false}; auto name = ModuleName{"", "data_json"}; auto consumer = [&as_expected](auto values) { if ((*values[0])["foo"] == "bar") { as_expected = true; }; }; SECTION("via file") { CHECK(ReadJsonFile( "foo.json", name, consumer, nullptr, /*use_git=*/false)); CHECK(as_expected); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadJsonFile("foo.json", name, consumer, &storage_config.Get(), /*use_git=*/true)); CHECK(as_expected); } } TEST_CASE("non existent") { bool as_expected{false}; std::atomic failcont_counter{0}; auto consumer = [&as_expected](auto values) { // Missing optional files are expected to result in empty objects with // no entries in it. if (values[0]->is_object() and values[0]->empty()) { as_expected = true; }; }; auto fail_func = [&failcont_counter]() { ++failcont_counter; }; SECTION("optional") { auto name = ModuleName{"", "missing"}; SECTION("via file") { CHECK(ReadJsonFile("foo.json", name, consumer, nullptr, /*use_git=*/false, fail_func)); CHECK(as_expected); CHECK(failcont_counter == 0); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadJsonFile("foo.json", name, consumer, &storage_config.Get(), /*use_git=*/true, fail_func)); CHECK(as_expected); CHECK(failcont_counter == 0); } } SECTION("mandatory") { auto name = ModuleName{"", "missing"}; SECTION("via file") { CHECK_FALSE(ReadJsonFile("foo.json", name, consumer, nullptr, /*use_git=*/false, fail_func)); CHECK_FALSE(as_expected); CHECK(failcont_counter == 1); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK_FALSE(ReadJsonFile("foo.json", name, consumer, &storage_config.Get(), /*use_git=*/true, fail_func)); CHECK_FALSE(as_expected); CHECK(failcont_counter == 1); } } } TEST_CASE("Bad syntax") { std::atomic failcont_counter{0}; auto fail_func = [&failcont_counter]() { ++failcont_counter; }; CHECK_FALSE(ReadJsonFile( "bad.json", {"", "data_json"}, [](auto const& /* unused */) {}, nullptr, /*use_git=*/false, fail_func)); CHECK(failcont_counter == 1); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/rule_map.test.cpp000066400000000000000000000350511516554100600324430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include #include #include #include // std::move #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/build_engine/base_maps/test_repo.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { using namespace BuildMaps::Base; // NOLINT auto SetupConfig(StorageConfig const* storage_config, bool use_git) -> RepositoryConfig { auto root = FileRoot{kBasePath / "data_rule"}; if (use_git) { auto repo_path = CreateTestRepo(); REQUIRE(repo_path); REQUIRE(storage_config); auto git_root = FileRoot::FromGit(storage_config, *repo_path, kRuleTreeId); REQUIRE(git_root); root = std::move(*git_root); } RepositoryConfig repo_config{}; repo_config.SetInfo("", RepositoryConfig::RepositoryInfo{root}); return repo_config; } auto ReadUserRule(EntityName const& id, UserRuleMap::Consumer value_checker, StorageConfig const* storage_config = nullptr, bool use_git = false) -> bool { auto repo_config = SetupConfig(storage_config, use_git); auto expr_file_map = CreateExpressionFileMap(&repo_config, 0); auto expr_func_map = CreateExpressionMap(&expr_file_map, &repo_config); auto rule_file_map = CreateRuleFileMap(&repo_config, 0); auto user_rule_map = CreateRuleMap(&rule_file_map, &expr_func_map, &repo_config); bool success{true}; { TaskSystem ts; user_rule_map.ConsumeAfterKeysReady( &ts, {id}, std::move(value_checker), [&success](std::string const& /*unused*/, bool /*unused*/) { success = false; }); } return success; } } // namespace TEST_CASE("Test empty rule", "[expression_map]") { auto name = EntityName{"", ".", "test_empty_rule"}; auto consumer = [](auto values) { REQUIRE(values[0]); }; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Test rule fields", "[rule_map]") { auto name = EntityName{"", ".", "test_rule_fields"}; auto consumer = [](auto values) { REQUIRE(*values[0]); REQUIRE_FALSE((*values[0])->StringFields().empty()); REQUIRE_FALSE((*values[0])->TargetFields().empty()); REQUIRE_FALSE((*values[0])->ConfigFields().empty()); CHECK((*values[0])->StringFields()[0] == "foo"); CHECK((*values[0])->TargetFields()[0] == "bar"); CHECK((*values[0])->ConfigFields()[0] == "baz"); }; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Test config_transitions target", "[rule_map]") { auto consumer = [](auto values) { REQUIRE(*values[0]); }; SECTION("via field") { auto name = EntityName{"", ".", "test_config_transitions_target_via_field"}; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } SECTION("via implicit") { auto name = EntityName{"", ".", "test_config_transitions_target_via_implicit"}; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } } TEST_CASE("Test config_transitions canonicalness", "[rule_map]") { auto name = EntityName{"", ".", "test_config_transitions_canonicalness"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto const& transitions = (*values[0])->ConfigTransitions(); REQUIRE(transitions.size() == 4); REQUIRE(transitions.at("foo")); REQUIRE(transitions.at("bar")); REQUIRE(transitions.at("baz")); REQUIRE(transitions.at("qux")); auto foo = transitions.at("foo")->Evaluate({}, {}); auto bar = transitions.at("bar")->Evaluate({}, {}); auto baz = transitions.at("baz")->Evaluate({}, {}); auto qux = transitions.at("qux")->Evaluate({}, {}); CHECK(foo == Expression::FromJson(R"([{}])"_json)); CHECK(bar == Expression::FromJson(R"([{"exists": true}])"_json)); CHECK(baz == Expression::FromJson(R"([{}])"_json)); CHECK(qux == Expression::FromJson(R"([{"defined": true}])"_json)); }; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Test call of imported expression", "[rule_map]") { auto name = EntityName{"", ".", "test_call_import"}; auto consumer = [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0])->Expression(); REQUIRE(expr); auto result = expr->Evaluate( Configuration{Expression::FromJson(R"({"FOO": "bar"})"_json)}, {}); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result["type"] == Expression{std::string{"RESULT"}}); CHECK(result["artifacts"] == Expression::FromJson(R"({"foo": "bar"})"_json)); }; SECTION("via file") { CHECK(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Fail due to unknown ID", "[rule_map]") { auto name = EntityName{"", ".", "does_not_exist"}; auto consumer = [](auto /*values*/) { CHECK(false); // should never be called }; SECTION("via file") { CHECK_FALSE(ReadUserRule(name, consumer, nullptr, /*use_git=*/false)); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK_FALSE(ReadUserRule( name, consumer, &storage_config.Get(), /*use_git=*/true)); } } TEST_CASE("Fail due to conflicting keyword names", "[rule_map]") { SECTION("string_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_string_kw_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("target_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_target_kw_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("config_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_config_kw_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("implicit_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_implicit_kw_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } } TEST_CASE("Fail due to conflicting field names", "[rule_map]") { SECTION("string <-> target") { CHECK_FALSE(ReadUserRule({"", ".", "test_string_target_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("target <-> config") { CHECK_FALSE(ReadUserRule({"", ".", "test_target_config_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("config <-> implicit") { CHECK_FALSE(ReadUserRule({"", ".", "test_config_implicit_conflict"}, [](auto /*values*/) { CHECK(false); // should never be called })); } } TEST_CASE("Fail due to unknown config_transitions target", "[rule_map]") { CHECK_FALSE( ReadUserRule({"", ".", "test_unknown_config_transitions_target"}, [](auto /*values*/) { CHECK(false); // should never be called })); } TEST_CASE("missing config_vars", "[rule_map]") { CHECK(ReadUserRule({"", ".", "test_missing_config_vars"}, [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0])->Expression(); REQUIRE(expr); auto result = expr->Evaluate( Configuration{Expression::FromJson(R"({"FOO": "bar"})"_json)}, {}); CHECK(result["artifacts"]["foo"] == Expression::FromJson(R"(null)"_json)); })); } TEST_CASE("Fail due to missing imports", "[rule_map]") { CHECK(ReadUserRule({"", ".", "test_missing_imports"}, [](auto values) { REQUIRE(*values[0]); auto expr = (*values[0])->Expression(); REQUIRE(expr); auto result = expr->Evaluate( Configuration{Expression::FromJson(R"({"FOO": "bar"})"_json)}, {}); CHECK_FALSE(result); })); } TEST_CASE("Malformed rule description", "[rule_map]") { SECTION("Malformed rule") { CHECK_FALSE( ReadUserRule({"", ".", "test_malformed_rule"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed rule expression") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_rule_expression"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed target_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_target_fields"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed string_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_string_fields"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed config_fields") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_config_fields"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed implicit") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_implicit"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed implicit entry") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_implicit_entry"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed implicit entity name") { CHECK_FALSE( ReadUserRule({"", ".", "test_malformed_implicit_entity_name"}, [](auto /*values*/) { CHECK(false); // should never be called })); CHECK_FALSE( ReadUserRule({"", ".", "test_malformed_implicit_entity_name_2"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed config_vars") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_config_vars"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed config_transitions") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_config_transitions"}, [](auto /*values*/) { CHECK(false); // should never be called })); } SECTION("Malformed imports") { CHECK_FALSE(ReadUserRule({"", ".", "test_malformed_imports"}, [](auto /*values*/) { CHECK(false); // should never be called })); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/source_map.test.cpp000066400000000000000000000216451516554100600330000ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/base_maps/source_map.hpp" #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/build_engine/base_maps/test_repo.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { using namespace BuildMaps::Base; // NOLINT auto SetupConfig(StorageConfig const* storage_config, bool use_git) -> RepositoryConfig { // manually create locally a test symlink in data_src; should match the // git test_repo structure if (not use_git) { auto link_path = kBasePath / "data_src/foo/link"; if (not FileSystemManager::Exists(link_path)) { REQUIRE(FileSystemManager::CreateSymlink("dummy", link_path)); } } auto root = FileRoot{kBasePath / "data_src"}; if (use_git) { auto repo_path = CreateTestRepo(); REQUIRE(repo_path); REQUIRE(storage_config); auto git_root = FileRoot::FromGit(storage_config, *repo_path, kSrcTreeId); REQUIRE(git_root); root = std::move(*git_root); } RepositoryConfig repo_config{}; repo_config.SetInfo("", RepositoryConfig::RepositoryInfo{root}); return repo_config; } auto ReadSourceTarget(EntityName const& id, SourceTargetMap::Consumer consumer, HashFunction::Type hash_type, StorageConfig const* storage_config, bool use_git = false, std::optional fail_func = std::nullopt) -> bool { auto repo_config = SetupConfig(storage_config, use_git); auto directory_entries = CreateDirectoryEntriesMap(&repo_config); auto source_artifacts = CreateSourceTargetMap(&directory_entries, &repo_config, hash_type); std::string error_msg; bool success{true}; { TaskSystem ts; source_artifacts.ConsumeAfterKeysReady( &ts, {id}, std::move(consumer), [&success, &error_msg](std::string const& msg, bool /*unused*/) { success = false; error_msg = msg; }, fail_func ? std::move(*fail_func) : [] {}); } return success and error_msg.empty(); } } // namespace TEST_CASE("from file") { auto const hash_type = TestHashType::ReadFromEnvironment(); nlohmann::json artifacts; auto name = EntityName{"", ".", "file"}; auto consumer = [&artifacts](auto values) { artifacts = (*values[0])->Artifacts()->ToJson(); }; SECTION("via file") { CHECK(ReadSourceTarget( name, consumer, hash_type, nullptr, /*use_git=*/false)); CHECK(artifacts["file"]["type"] == "LOCAL"); CHECK(artifacts["file"]["data"]["path"] == "file"); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadSourceTarget(name, consumer, hash_type, &storage_config.Get(), /*use_git=*/true)); CHECK(artifacts["file"]["type"] == "KNOWN"); CHECK( artifacts["file"]["data"]["id"] == (ProtocolTraits::IsNative(hash_type) ? kEmptySha1 : kEmptySha256)); CHECK(artifacts["file"]["data"]["size"] == 0); } } TEST_CASE("not present at all") { auto const hash_type = TestHashType::ReadFromEnvironment(); bool consumed{false}; bool failure_called{false}; auto name = EntityName{"", ".", "does_not_exist"}; auto consumer = [&consumed](auto /*unused*/) { consumed = true; }; auto fail_func = [&failure_called]() { failure_called = true; }; SECTION("via file") { CHECK_FALSE(ReadSourceTarget( name, consumer, hash_type, nullptr, /*use_git=*/false, fail_func)); CHECK_FALSE(consumed); CHECK(failure_called); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK_FALSE(ReadSourceTarget(name, consumer, hash_type, &storage_config.Get(), /*use_git=*/true, fail_func)); CHECK_FALSE(consumed); CHECK(failure_called); } } TEST_CASE("malformed entry") { auto const hash_type = TestHashType::ReadFromEnvironment(); bool consumed{false}; bool failure_called{false}; auto name = EntityName{"", ".", "bad_entry"}; auto consumer = [&consumed](auto /*unused*/) { consumed = true; }; auto fail_func = [&failure_called]() { failure_called = true; }; SECTION("via git tree") { CHECK_FALSE(ReadSourceTarget( name, consumer, hash_type, nullptr, /*use_git=*/false, fail_func)); CHECK_FALSE(consumed); CHECK(failure_called); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK_FALSE(ReadSourceTarget(name, consumer, hash_type, &storage_config.Get(), /*use_git=*/true, fail_func)); CHECK_FALSE(consumed); CHECK(failure_called); } } TEST_CASE("subdir file") { auto const hash_type = TestHashType::ReadFromEnvironment(); nlohmann::json artifacts; auto name = EntityName{"", "foo", "bar/file"}; auto consumer = [&artifacts](auto values) { artifacts = (*values[0])->Artifacts()->ToJson(); }; SECTION("via file") { CHECK(ReadSourceTarget( name, consumer, hash_type, nullptr, /*use_git=*/false)); CHECK(artifacts["bar/file"]["type"] == "LOCAL"); CHECK(artifacts["bar/file"]["data"]["path"] == "foo/bar/file"); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadSourceTarget(name, consumer, hash_type, &storage_config.Get(), /*use_git=*/true)); CHECK(artifacts["bar/file"]["type"] == "KNOWN"); CHECK( artifacts["bar/file"]["data"]["id"] == (ProtocolTraits::IsNative(hash_type) ? kEmptySha1 : kEmptySha256)); CHECK(artifacts["bar/file"]["data"]["size"] == 0); } } TEST_CASE("subdir symlink") { auto const hash_type = TestHashType::ReadFromEnvironment(); nlohmann::json artifacts; auto name = EntityName{"", "foo", "link"}; auto consumer = [&artifacts](auto values) { artifacts = (*values[0])->Artifacts()->ToJson(); }; SECTION("via file") { CHECK(ReadSourceTarget( name, consumer, hash_type, nullptr, /*use_git=*/false)); CHECK(artifacts["link"]["type"] == "LOCAL"); CHECK(artifacts["link"]["data"]["path"] == "foo/link"); } SECTION("via git tree") { auto const storage_config = TestStorageConfig::Create(); CHECK(ReadSourceTarget(name, consumer, hash_type, &storage_config.Get(), /*use_git=*/true)); CHECK(artifacts["link"]["type"] == "KNOWN"); CHECK(artifacts["link"]["data"]["id"] == (ProtocolTraits::IsNative(hash_type) ? kSrcLinkIdSha1 : kSrcLinkIdSha256)); CHECK(artifacts["link"]["data"]["size"] == 5); // content: dummy } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/base_maps/test_repo.hpp000066400000000000000000000054131516554100600316710ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TEST_REPO_HPP #define INCLUDED_SRC_TEST_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TEST_REPO_HPP #include #include #include #include #include #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "test/utils/shell_quoting.hpp" static auto const kBasePath = std::filesystem::path{"test/buildtool/build_engine/base_maps"}; static auto const kBundlePath = kBasePath / "data/test_repo.bundle"; static auto const kSrcTreeId = std::string{"6d57ba31821f69286e280334e4fd5f9dbd141721"}; static auto const kSrcLinkIdSha1 = std::string{"2995a4d0e74917fd3e1383c577d0fc301fff1b04"}; static auto const kSrcLinkIdSha256 = std::string{ "b5a2c96250612366ea272ffac6d9744aaf4b45aacd96aa7cfcb931ee3b558259"}; static auto const kRuleTreeId = std::string{"c6dd902c9d4e7afa8b20eb04e58503e63ecab84d"}; static auto const kExprTreeId = std::string{"4946bd21d0a5b3e0c82d6944f3d47adaf1bb66f7"}; static auto const kJsonTreeId = std::string{"6982563dfc4dcdd1362792dbbc9d8243968d1ec9"}; static auto const kEmptySha1 = std::string{"e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}; static auto const kEmptySha256 = std::string{ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}; [[nodiscard]] static inline auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / kBasePath; } [[nodiscard]] static inline auto CreateTestRepo() -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone --bare {} {}", QuoteForShell(kBundlePath.string()), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } #endif // INCLUDED_SRC_TEST_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TEST_REPO_HPP just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/expression/000077500000000000000000000000001516554100600274165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/expression/TARGETS000066400000000000000000000032611516554100600304540ustar00rootroot00000000000000{ "linked_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["linked_map"] , "srcs": ["linked_map.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/build_engine/expression", "linked_map"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "expression"] } , "expression": { "type": ["@", "rules", "CC/test", "test"] , "name": ["expression"] , "srcs": ["expression.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , ["@", "src", "src/buildtool/build_engine/expression", "linked_map"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "expression"] } , "configuration": { "type": ["@", "rules", "CC/test", "test"] , "name": ["configuration"] , "srcs": ["configuration.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "expression"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["expression"] , "deps": ["configuration", "expression", "linked_map"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/expression/configuration.test.cpp000066400000000000000000000123471516554100600337560ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/configuration.hpp" #include #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" TEST_CASE("Access", "[configuration]") { auto env = Configuration{Expression::FromJson(R"({"foo": 1, "bar": 2})"_json)}; CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env[Expression::FromJson(R"("bar")"_json)] == Expression::FromJson("2"_json)); CHECK(env["baz"] == Expression::FromJson(R"(null)"_json)); CHECK(env[Expression::FromJson(R"("baz")"_json)] == Expression::FromJson(R"(null)"_json)); } TEST_CASE("Update", "[configuration]") { SECTION("Append") { auto env = Configuration{Expression::FromJson(R"({})"_json)}; env = env.Update(Expression::FromJson(R"({"foo": 1})"_json)); CHECK(env["foo"] == Expression::FromJson("1"_json)); env = env.Update("bar", Expression::number_t{2}); CHECK(env["bar"] == Expression::FromJson("2"_json)); env = env.Update(Expression::map_t::underlying_map_t{ {"baz", ExpressionPtr{Expression::number_t{3}}}}); CHECK(env["baz"] == Expression::FromJson("3"_json)); } SECTION("Overwrite") { auto env = Configuration{ Expression::FromJson(R"({"foo": 1, "bar": 2, "baz" : 3})"_json)}; CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); CHECK(env["baz"] == Expression::FromJson("3"_json)); env = env.Update(Expression::FromJson(R"({"foo": 10})"_json)); CHECK(env["foo"] == Expression::FromJson("10"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); CHECK(env["baz"] == Expression::FromJson("3"_json)); env = env.Update("bar", Expression::number_t{20}); // NOLINT CHECK(env["foo"] == Expression::FromJson("10"_json)); CHECK(env["bar"] == Expression::FromJson("20"_json)); CHECK(env["baz"] == Expression::FromJson("3"_json)); env = env.Update(Expression::map_t::underlying_map_t{ {"baz", ExpressionPtr{Expression::number_t{30}}}}); // NOLINT CHECK(env["foo"] == Expression::FromJson("10"_json)); CHECK(env["bar"] == Expression::FromJson("20"_json)); CHECK(env["baz"] == Expression::FromJson("30"_json)); } } TEST_CASE("Prune", "[configuration]") { auto env = Configuration{Expression::FromJson(R"({"foo": 1, "bar": 2})"_json)}; CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); SECTION("Via string list") { env = env.Prune(std::vector{"foo", "bar", "baz"}); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); env = env.Prune(std::vector{"foo", "bar"}); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); env = env.Prune(std::vector{"foo"}); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson(R"(null)"_json)); env = env.Prune(std::vector{}); CHECK(env["foo"] == Expression::FromJson(R"(null)"_json)); CHECK(env["bar"] == Expression::FromJson(R"(null)"_json)); } SECTION("Via expression") { env = env.Prune(Expression::FromJson(R"(["foo", "bar", "baz"])"_json)); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); env = env.Prune(Expression::FromJson(R"(["foo", "bar"])"_json)); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson("2"_json)); env = env.Prune(Expression::FromJson(R"(["foo"])"_json)); CHECK(env["foo"] == Expression::FromJson("1"_json)); CHECK(env["bar"] == Expression::FromJson(R"(null)"_json)); env = env.Prune(Expression::FromJson(R"([])"_json)); CHECK(env["foo"] == Expression::FromJson(R"(null)"_json)); CHECK(env["bar"] == Expression::FromJson(R"(null)"_json)); CHECK_THROWS_AS(env.Prune(Expression::FromJson( R"(["not_all_string", false])"_json)), Expression::ExpressionTypeError); CHECK_THROWS_AS(env.Prune(Expression::FromJson(R"("not_a_list")"_json)), Expression::ExpressionTypeError); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/expression/expression.test.cpp000066400000000000000000002233631516554100600333100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/expression.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/function_map.hpp" #include "src/buildtool/build_engine/expression/linked_map.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact_description.hpp" TEST_CASE("Expression access", "[expression]") { // NOLINT using namespace std::string_literals; using path = std::filesystem::path; using none_t = Expression::none_t; using number_t = Expression::number_t; using artifact_t = Expression::artifact_t; using result_t = Expression::result_t; using list_t = Expression::list_t; using map_t = Expression::map_t; auto none = ExpressionPtr{}; auto boolean = ExpressionPtr{true}; auto number = ExpressionPtr{number_t{1}}; auto string = ExpressionPtr{"2"s}; auto artifact = ExpressionPtr{ArtifactDescription::CreateTree(path{"local_path"})}; auto result = ExpressionPtr{result_t{boolean, number, string}}; auto list = ExpressionPtr{list_t{number}}; auto map = ExpressionPtr{map_t{{"3"s, number}}}; SECTION("Type checks") { CHECK(none->IsNone()); CHECK(boolean->IsBool()); CHECK_FALSE(boolean->IsNone()); CHECK(number->IsNumber()); CHECK_FALSE(number->IsNone()); CHECK(string->IsString()); CHECK_FALSE(string->IsNone()); CHECK(artifact->IsArtifact()); CHECK_FALSE(artifact->IsNone()); CHECK(result->IsResult()); CHECK_FALSE(result->IsNone()); CHECK(list->IsList()); CHECK_FALSE(list->IsNone()); CHECK(map->IsMap()); CHECK_FALSE(map->IsNone()); } SECTION("Throwing accessors") { CHECK(boolean->Bool() == true); CHECK_THROWS_AS(boolean->Number(), Expression::ExpressionTypeError); CHECK(number->Number() == number_t{1}); CHECK_THROWS_AS(number->Bool(), Expression::ExpressionTypeError); CHECK(string->String() == "2"s); CHECK_THROWS_AS(string->Artifact(), Expression::ExpressionTypeError); CHECK(artifact->Artifact() == ArtifactDescription::CreateTree(path{"local_path"})); CHECK_THROWS_AS(artifact->String(), Expression::ExpressionTypeError); CHECK(result->Result() == result_t{boolean, number, string}); CHECK_THROWS_AS(result->String(), Expression::ExpressionTypeError); CHECK_THAT(list->List(), Catch::Matchers::Equals({number})); CHECK_THROWS_AS(list->Map(), Expression::ExpressionTypeError); REQUIRE(map->Map().at("3"s) == number); CHECK_THROWS_AS(map->List(), Expression::ExpressionTypeError); } SECTION("Non-throwing accessors") { CHECK(none->Value()); CHECK(boolean->Value()); CHECK_FALSE(boolean->Value()); CHECK(number->Value()); CHECK_FALSE(number->Value()); CHECK(string->Value()); CHECK_FALSE(string->Value()); CHECK(artifact->Value()); CHECK_FALSE(artifact->Value()); CHECK(result->Value()); CHECK_FALSE(result->Value()); CHECK(list->Value()); CHECK_FALSE(list->Value()); CHECK(map->Value()); CHECK_FALSE(map->Value()); } SECTION("Non-throwing comparison operator") { CHECK(none == none); CHECK(none == Expression{}); CHECK(none == Expression::FromJson("null"_json)); CHECK(none != Expression{false}); CHECK(none != Expression{number_t{0}}); CHECK(none != Expression{""s}); CHECK(none != Expression{"0"s}); CHECK(none != Expression{list_t{}}); CHECK(none != Expression{map_t{}}); CHECK(boolean == boolean); CHECK(boolean == true); CHECK(boolean == Expression{true}); CHECK(boolean == Expression::FromJson("true"_json)); CHECK(boolean != false); CHECK(boolean != Expression{false}); CHECK(boolean != number_t{1}); CHECK(boolean != number); CHECK(boolean != Expression::FromJson("false"_json)); CHECK(number == number); CHECK(number == number_t{1}); CHECK(number == Expression{number_t{1}}); CHECK(number == Expression::FromJson("1"_json)); CHECK(number != number_t{}); CHECK(number != Expression{number_t{}}); CHECK(number != true); CHECK(number != boolean); CHECK(number != Expression::FromJson("0"_json)); CHECK(string == string); CHECK(string == "2"s); CHECK(string == Expression{"2"s}); CHECK(string == Expression::FromJson(R"("2")"_json)); CHECK(string != ""s); CHECK(string != Expression{""s}); CHECK(string != ArtifactDescription::CreateTree(path{"local_path"})); CHECK(string != artifact); CHECK(string != Expression::FromJson(R"("")"_json)); CHECK(artifact == artifact); CHECK(artifact == ArtifactDescription::CreateTree(path{"local_path"})); CHECK(artifact == Expression{ArtifactDescription::CreateTree(path{"local_path"})}); CHECK(artifact != ""s); CHECK(artifact != string); CHECK(result == result); CHECK(result == result_t{boolean, number, string}); CHECK(result == Expression{result_t{boolean, number, string}}); CHECK(result != ""s); CHECK(result != string); CHECK(list == list); CHECK(list == list_t{number}); CHECK(list == Expression{list_t{number}}); CHECK(list == Expression::FromJson("[1]"_json)); CHECK(list != list_t{}); CHECK(list != Expression{list_t{}}); CHECK(list != map); CHECK(list != *map); CHECK(list != Expression::FromJson(R"({"1":1})"_json)); CHECK(map == map); CHECK(map == map_t{{"3"s, number}}); CHECK(map == Expression{map_t{{"3"s, number}}}); CHECK(map == Expression::FromJson(R"({"3":1})"_json)); CHECK(map != map_t{}); CHECK(map != Expression{map_t{}}); CHECK(map != list); CHECK(map != *list); CHECK(map != Expression::FromJson(R"(["3",1])"_json)); // compare nullptr != null != false != 0 != "" != [] != {} auto exprs = std::vector{ ExpressionPtr{nullptr}, ExpressionPtr{ArtifactDescription::CreateTree(path{""})}, ExpressionPtr{result_t{}}, Expression::FromJson("null"_json), Expression::FromJson("false"_json), Expression::FromJson("0"_json), Expression::FromJson(R"("")"_json), Expression::FromJson("[]"_json), Expression::FromJson("{}"_json)}; for (auto const& l : exprs) { for (auto const& r : exprs) { if (&l != &r) { CHECK(l != r); } } } } SECTION("Throwing access operator") { // operators with argument of type size_t expect list CHECK_THROWS_AS(none[0], Expression::ExpressionTypeError); CHECK_THROWS_AS(boolean[0], Expression::ExpressionTypeError); CHECK_THROWS_AS(number[0], Expression::ExpressionTypeError); CHECK_THROWS_AS(string[0], Expression::ExpressionTypeError); CHECK_THROWS_AS(artifact[0], Expression::ExpressionTypeError); CHECK_THROWS_AS(result[0], Expression::ExpressionTypeError); CHECK(list[0] == number); CHECK_THROWS_AS(map[0], Expression::ExpressionTypeError); // operators with argument of type std::string expect map CHECK_THROWS_AS(none["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(boolean["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(number["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(string["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(artifact["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(result["3"], Expression::ExpressionTypeError); CHECK_THROWS_AS(list["3"], Expression::ExpressionTypeError); CHECK(map["3"] == number); } } TEST_CASE("Expression from JSON", "[expression]") { auto none = Expression::FromJson("null"_json); REQUIRE(none); CHECK(none->IsNone()); auto boolean = Expression::FromJson("true"_json); REQUIRE(boolean); REQUIRE(boolean->IsBool()); CHECK(boolean->Bool() == true); auto number = Expression::FromJson("1"_json); REQUIRE(number); REQUIRE(number->IsNumber()); CHECK(number->Number() == 1); auto string = Expression::FromJson(R"("foo")"_json); REQUIRE(string); REQUIRE(string->IsString()); CHECK(string->String() == "foo"); auto list = Expression::FromJson("[]"_json); REQUIRE(list); REQUIRE(list->IsList()); CHECK(list->List().empty()); auto map = Expression::FromJson("{}"_json); REQUIRE(map); REQUIRE(map->IsMap()); CHECK(map->Map().empty()); } namespace { auto TestToJson(nlohmann::json const& json) -> void { auto expr = Expression::FromJson(json); REQUIRE(expr); CHECK(expr->ToJson() == json); } } // namespace TEST_CASE("Expression to JSON", "[expression]") { TestToJson("null"_json); TestToJson("true"_json); TestToJson("1"_json); TestToJson(R"("foo")"_json); TestToJson("[]"_json); TestToJson("{}"_json); } namespace { template concept ValidExpressionTypeOrPtr = Expression::IsValidType() or std::is_same_v; template auto Add(ExpressionPtr const& expr, std::string const& key, T const& by) -> ExpressionPtr { try { auto new_map = Expression::map_t::underlying_map_t{}; new_map.emplace(key, by); return ExpressionPtr{Expression::map_t{expr, new_map}}; } catch (...) { return ExpressionPtr{nullptr}; } } template auto Replace(ExpressionPtr const& expr, std::string const& key, T const& by) -> ExpressionPtr { auto const& map = expr->Map(); if (not map.contains(key)) { return ExpressionPtr{nullptr}; } return Add(expr, key, by); } } // namespace TEST_CASE("Expression Evaluation", "[expression]") { // NOLINT using namespace std::string_literals; using number_t = Expression::number_t; using list_t = Expression::list_t; auto env = Configuration{}; auto fcts = FunctionMapPtr{}; auto foo = ExpressionPtr{"foo"s}; auto bar = ExpressionPtr{"bar"s}; auto baz = ExpressionPtr{"baz"s}; SECTION("list object") { auto expr = Expression::FromJson(R"(["foo", "bar", "baz"])"_json); REQUIRE(expr); REQUIRE(expr->IsList()); CHECK(expr->List().size() == 3); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result->List().size() == 3); CHECK(*result == *expr); } SECTION("map object without type") { auto expr = Expression::FromJson(R"({"foo": "bar"})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); CHECK_FALSE(result); } SECTION("custom function") { auto expr = Expression::FromJson(R"( { "type": "'" , "$1": "PLACEHOLDER" })"_json); REQUIRE(expr); auto literal = Expression::FromJson(R"({"foo": "bar"})"_json); REQUIRE(literal); expr = Replace(expr, "$1", literal); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); CHECK(result); CHECK(*result == *literal); } SECTION("var expression") { auto expr = Expression::FromJson(R"( { "type": "var" , "name": "foo" })"_json); REQUIRE(expr); auto none_result = expr.Evaluate(env, fcts); CHECK(none_result == Expression::FromJson(R"(null)"_json)); env = env.Update("foo", "bar"s); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("bar")"_json)); auto overwrite = expr.Evaluate(env.Update("foo", list_t{result}), fcts); REQUIRE(overwrite); REQUIRE(overwrite->IsList()); CHECK(overwrite == Expression::FromJson(R"(["bar"])"_json)); } SECTION("quote expression") { auto expr = Expression::FromJson(R"( {"type": "'", "$1": {"type": "var", "name": "this is literal"}} )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); CHECK(result == Expression::FromJson(R"( {"type": "var", "name": "this is literal"})"_json)); auto expr_empty = Expression::FromJson(R"({"type": "'"})"_json); REQUIRE(expr_empty); auto result_empty = expr_empty.Evaluate(env, fcts); CHECK(result_empty == Expression::FromJson(R"(null)"_json)); } SECTION("quasi-quote expression") { auto expr = Expression::FromJson(R"({"type": "`", "$1": { "foo": {"type": ",", "$1": {"type": "var", "name": "foo_var"}} , "bar": [ 1, 2, ["deep", "literals"] , {"type": ",@", "$1": {"type": "var", "name": "bar_var"}} , 3 , {"type": ",", "$1": {"type": "var", "name": "bar_plain"}} , 4, 5 , {"type": ",", "$1": {"type": "var", "name": "foo_var"}} , [ "deep", "expansion" , {"type": ",", "$1": {"type": "var", "name": "bar_plain"}} , {"type": ",@", "$1": {"type": "var", "name": "bar_var"}} , {"type": ",", "$1": {"type": "var", "name": "foo_var"}} ] ] } })"_json); REQUIRE(expr); env = env.Update("foo_var", "foo value"s); env = env.Update("bar_var", Expression::FromJson(R"(["b", "a", "r"])"_json)); env = env.Update("bar_plain", Expression::FromJson(R"(["kept", "as", "list"])"_json)); auto result = expr.Evaluate(env, fcts); auto expected = Expression::FromJson(R"( { "foo": "foo value" , "bar": [ 1, 2, ["deep", "literals"] , "b", "a", "r" , 3 , ["kept", "as", "list"] , 4, 5 , "foo value" , [ "deep", "expansion" , ["kept", "as", "list"] , "b", "a", "r" , "foo value" ] ] })"_json); CHECK(result == expected); auto doc_example_a = Expression::FromJson( R"({"type": "`", "$1": [1, 2, {"type": ",@", "$1": [3, 4]}]})"_json); auto result_a = doc_example_a.Evaluate(env, fcts); CHECK(result_a == Expression::FromJson(R"([1, 2, 3, 4])"_json)); auto doc_example_b = Expression::FromJson( R"({"type": "`", "$1": [1, 2, {"type": ",", "$1": [3, 4]}]})"_json); auto result_b = doc_example_b.Evaluate(env, fcts); CHECK(result_b == Expression::FromJson(R"([1, 2, [3, 4]])"_json)); } SECTION("if expression") { auto expr = Expression::FromJson(R"( { "type": "if" , "cond": "PLACEHOLDER" , "then": "success" , "else": "failure" })"_json); REQUIRE(expr); SECTION("Boolean condition") { expr = Replace(expr, "cond", true); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsString()); CHECK(success == Expression::FromJson(R"("success")"_json)); expr = Replace(expr, "cond", false); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsString()); CHECK(failure == Expression::FromJson(R"("failure")"_json)); } SECTION("Number condition") { expr = Replace(expr, "cond", number_t{1}); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsString()); CHECK(success == Expression::FromJson(R"("success")"_json)); expr = Replace(expr, "cond", number_t{0}); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsString()); CHECK(failure == Expression::FromJson(R"("failure")"_json)); } SECTION("String condition") { expr = Replace(expr, "cond", "false"s); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsString()); CHECK(success == Expression::FromJson(R"("success")"_json)); expr = Replace(expr, "cond", ""s); REQUIRE(expr); auto fail1 = expr.Evaluate(env, fcts); REQUIRE(fail1); REQUIRE(fail1->IsString()); CHECK(fail1 == Expression::FromJson(R"("failure")"_json)); } SECTION("List condition") { expr = Replace(expr, "cond", list_t{ExpressionPtr{}}); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsString()); CHECK(success == Expression::FromJson(R"("success")"_json)); expr = Replace(expr, "cond", list_t{}); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsString()); CHECK(failure == Expression::FromJson(R"("failure")"_json)); } SECTION("Map condition") { auto literal = Expression::FromJson( R"({"type": "'", "$1": {"foo": "bar"}})"_json); REQUIRE(literal); expr = Replace(expr, "cond", literal); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsString()); CHECK(success == Expression::FromJson(R"("success")"_json)); auto empty = Expression::FromJson(R"({"type": "'", "$1": {}})"_json); REQUIRE(empty); expr = Replace(expr, "cond", empty); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsString()); CHECK(failure == Expression::FromJson(R"("failure")"_json)); } } SECTION("if defaults") { auto expr = Expression::FromJson(R"( { "type": "if" , "cond": {"type": "var", "name": "x"} } )"_json); CHECK(expr.Evaluate(env.Update("x", true), fcts) == Expression::FromJson(R"([])"_json)); CHECK(expr.Evaluate(env.Update("x", false), fcts) == Expression::FromJson(R"([])"_json)); } SECTION("cond expression") { auto expr = Expression::FromJson(R"( { "type": "cond" , "cond": [ [ { "type": "==" , "$1": {"type":"var", "name": "val", "default": ""} , "$2": 0 } , "number" ] , [ { "type": "==" , "$1": {"type":"var", "name": "val", "default": ""} , "$2": "0" } , "string" ] , [ { "type": "==" , "$1": {"type":"var", "name": "val", "default": ""} , "$2": false } , "boolean" ] , [ {"type":"var", "name": "val", "default": ""}, "first" ] , [ {"type":"var", "name": "val", "default": ""}, "second" ] ]})"_json); REQUIRE(expr); auto number = expr.Evaluate(env.Update("val", 0.0), fcts); REQUIRE(number); REQUIRE(number->IsString()); CHECK(number == Expression::FromJson(R"("number")"_json)); auto string = expr.Evaluate(env.Update("val", "0"s), fcts); REQUIRE(string); REQUIRE(string->IsString()); CHECK(string == Expression::FromJson(R"("string")"_json)); auto boolean = expr.Evaluate(env.Update("val", false), fcts); REQUIRE(boolean); REQUIRE(boolean->IsString()); CHECK(boolean == Expression::FromJson(R"("boolean")"_json)); auto first = expr.Evaluate(env.Update("val", true), fcts); REQUIRE(first); REQUIRE(first->IsString()); CHECK(first == Expression::FromJson(R"("first")"_json)); auto default1 = expr.Evaluate(env, fcts); REQUIRE(default1); REQUIRE(default1->IsList()); CHECK(default1 == Expression::FromJson(R"([])"_json)); expr = Add(expr, "default", "default"s); auto default2 = expr.Evaluate(env, fcts); REQUIRE(default2); REQUIRE(default2->IsString()); CHECK(default2 == Expression::FromJson(R"("default")"_json)); } SECTION("case expression") { auto expr = Expression::FromJson(R"( { "type": "case" , "expr": {"type": "var", "name": "val", "default": ""} , "case": { "foo": "FOO" , "bar": {"type": "var", "name": "bar", "default": "BAR"} } })"_json); REQUIRE(expr); auto foo = expr.Evaluate(env.Update("val", "foo"s), fcts); REQUIRE(foo); REQUIRE(foo->IsString()); CHECK(foo == Expression::FromJson(R"("FOO")"_json)); auto bar = expr.Evaluate(env.Update("val", "bar"s), fcts); REQUIRE(bar); REQUIRE(bar->IsString()); CHECK(bar == Expression::FromJson(R"("BAR")"_json)); auto default1 = expr.Evaluate(env, fcts); REQUIRE(default1); REQUIRE(default1->IsList()); CHECK(default1 == Expression::FromJson(R"([])"_json)); expr = Add(expr, "default", "default"s); auto default2 = expr.Evaluate(env, fcts); REQUIRE(default2); REQUIRE(default2->IsString()); CHECK(default2 == Expression::FromJson(R"("default")"_json)); } SECTION("case* expression") { auto expr = Expression::FromJson(R"( { "type": "case*" , "expr": {"type": "var", "name": "val"} , "case": [ [false, "FOO"] , [ {"type": "var", "name": "bar", "default": null} , {"type": "var", "name": "bar", "default": "BAR"} ] , [0, {"type": "join", "$1": ["B", "A", "Z"]}] ] })"_json); REQUIRE(expr); auto foo = expr.Evaluate(env.Update("val", false), fcts); REQUIRE(foo); REQUIRE(foo->IsString()); CHECK(foo == Expression::FromJson(R"("FOO")"_json)); auto bar = expr.Evaluate(env, fcts); REQUIRE(bar); REQUIRE(bar->IsString()); CHECK(bar == Expression::FromJson(R"("BAR")"_json)); auto baz = expr.Evaluate(env.Update("val", 0.0), fcts); REQUIRE(baz); REQUIRE(baz->IsString()); CHECK(baz == Expression::FromJson(R"("BAZ")"_json)); auto default1 = expr.Evaluate(env.Update("val", ""s), fcts); REQUIRE(default1); REQUIRE(default1->IsList()); CHECK(default1 == Expression::FromJson(R"([])"_json)); expr = Add(expr, "default", "default"s); auto default2 = expr.Evaluate(env.Update("val", ""s), fcts); REQUIRE(default2); REQUIRE(default2->IsString()); CHECK(default2 == Expression::FromJson(R"("default")"_json)); } SECTION("== expression") { auto expr = Expression::FromJson(R"( { "type": "==" , "$1": "foo" , "$2": "PLACEHOLDER"})"_json); REQUIRE(expr); expr = Replace(expr, "$2", "foo"s); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsBool()); CHECK(success == Expression::FromJson("true"_json)); expr = Replace(expr, "$2", "bar"s); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsBool()); CHECK(failure == Expression::FromJson("false"_json)); } SECTION("not expression") { auto expr = Expression::FromJson(R"( { "type": "not" , "$1": {"type": "var", "name": "x" } })"_json); REQUIRE(expr); CHECK(expr.Evaluate(env.Update("x", true), fcts) == Expression::FromJson("false"_json)); CHECK(expr.Evaluate(env.Update("x", false), fcts) == Expression::FromJson("true"_json)); CHECK(expr.Evaluate(env.Update("x", Expression::FromJson(R"([])"_json)), fcts) == Expression::FromJson("true"_json)); CHECK(expr.Evaluate( env.Update("x", Expression::FromJson(R"(["a"])"_json)), fcts) == Expression::FromJson("false"_json)); CHECK(expr.Evaluate(env.Update("x", Expression::FromJson("null"_json)), fcts) == Expression::FromJson("true"_json)); CHECK(expr.Evaluate(env.Update("x", Expression::FromJson("0"_json)), fcts) == Expression::FromJson("true"_json)); CHECK(expr.Evaluate(env.Update("x", Expression::FromJson("1"_json)), fcts) == Expression::FromJson("false"_json)); CHECK(expr.Evaluate(env.Update("x", Expression::FromJson(R"("")"_json)), fcts) == Expression::FromJson("true"_json)); CHECK(expr.Evaluate( env.Update("x", Expression::FromJson(R"("0")"_json)), fcts) == Expression::FromJson("false"_json)); } SECTION("and expression") { auto expr = Expression::FromJson(R"( { "type": "and" , "$1": "PLACEHOLDER" })"_json); REQUIRE(expr); auto empty = ExpressionPtr{""s}; expr = Replace(expr, "$1", list_t{foo, bar}); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsBool()); CHECK(success == Expression::FromJson("true"_json)); expr = Replace(expr, "$1", list_t{foo, empty}); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsBool()); CHECK(failure == Expression::FromJson("false"_json)); // test evaluation of list elements expr = Replace(expr, "$1", list_t{foo, Expression::FromJson(R"( {"type": "'" , "$1": true})"_json)}); REQUIRE(expr); auto evaluated = expr.Evaluate(env, fcts); REQUIRE(evaluated); REQUIRE(evaluated->IsBool()); CHECK(evaluated == Expression::FromJson("true"_json)); // test short-circuit evaluation of logical and (static list) auto static_list = R"([true, false, {"type": "fail", "msg": "failed"}])"_json; expr = Replace(expr, "$1", Expression::FromJson(static_list)); REQUIRE(expr); auto static_result = expr.Evaluate(env, fcts); REQUIRE(static_result); REQUIRE(static_result->IsBool()); CHECK(static_result == Expression::FromJson("false"_json)); // test full evaluation of dynamic list (expression evaluating to list) auto dynamic_list = nlohmann::json{{"type", "context"}, {"$1", static_list}}; expr = Replace(expr, "$1", Expression::FromJson(dynamic_list)); REQUIRE(expr); auto dyn_result = expr.Evaluate(env, fcts); REQUIRE_FALSE(dyn_result); } SECTION("or expression") { auto expr = Expression::FromJson(R"( { "type": "or" , "$1": "PLACEHOLDER" })"_json); REQUIRE(expr); auto empty = ExpressionPtr{""s}; expr = Replace(expr, "$1", list_t{foo, bar}); REQUIRE(expr); auto success = expr.Evaluate(env, fcts); REQUIRE(success); REQUIRE(success->IsBool()); CHECK(success == Expression::FromJson("true"_json)); expr = Replace(expr, "$1", list_t{foo, empty}); REQUIRE(expr); auto failure = expr.Evaluate(env, fcts); REQUIRE(failure); REQUIRE(failure->IsBool()); CHECK(failure == Expression::FromJson("true"_json)); // test evaluation of list elements expr = Replace(expr, "$1", list_t{foo, Expression::FromJson(R"( {"type": "'" , "$1": true})"_json)}); REQUIRE(expr); auto evaluated = expr.Evaluate(env, fcts); REQUIRE(evaluated); REQUIRE(evaluated->IsBool()); CHECK(evaluated == Expression::FromJson("true"_json)); // test short-circuit evaluation of logical or (static list) auto static_list = R"([false, true, {"type": "fail", "msg": "failed"}])"_json; expr = Replace(expr, "$1", Expression::FromJson(static_list)); REQUIRE(expr); auto static_result = expr.Evaluate(env, fcts); REQUIRE(static_result); REQUIRE(static_result->IsBool()); CHECK(static_result == Expression::FromJson("true"_json)); // test full evaluation of dynamic list (expression evaluating to list) auto dynamic_list = nlohmann::json{{"type", "context"}, {"$1", static_list}}; expr = Replace(expr, "$1", Expression::FromJson(dynamic_list)); REQUIRE(expr); auto dyn_result = expr.Evaluate(env, fcts); REQUIRE_FALSE(dyn_result); } SECTION("++ expression") { auto expr = Expression::FromJson(R"( { "type": "++" , "$1": [ ["foo"] , ["bar", "baz"]]})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foo", "bar", "baz"])"_json)); } SECTION("+ expression") { auto expr_empty = Expression::FromJson(R"( { "type": "+" , "$1": [] })"_json); REQUIRE(expr_empty); auto result_empty = expr_empty.Evaluate(env, fcts); REQUIRE(result_empty); REQUIRE(result_empty->IsNumber()); CHECK(result_empty == Expression::FromJson(R"(0.0)"_json)); auto expr = Expression::FromJson(R"( { "type": "+" , "$1": [2, 3, 7, -1] })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsNumber()); CHECK(result == Expression::FromJson(R"(11.0)"_json)); } SECTION("* expression") { auto expr_empty = Expression::FromJson(R"( { "type": "*" , "$1": [] })"_json); REQUIRE(expr_empty); auto result_empty = expr_empty.Evaluate(env, fcts); REQUIRE(result_empty); REQUIRE(result_empty->IsNumber()); CHECK(result_empty == Expression::FromJson(R"(1.0)"_json)); auto expr = Expression::FromJson(R"( { "type": "*" , "$1": [2, 3, 7, -1] })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsNumber()); CHECK(result == Expression::FromJson(R"(-42.0)"_json)); } SECTION("nub_right expression") { auto expr = Expression::FromJson(R"( {"type": "nub_right" , "$1": ["-lfoo", "-lbar", "-lbaz", "-lbar"] })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["-lfoo", "-lbaz", "-lbar"])"_json)); } SECTION("nub_right expression 2") { auto expr = Expression::FromJson(R"( {"type": "nub_right" , "$1": { "type": "++" , "$1": [ ["libg.a"] , ["libe.a", "libd.a", "libc.a", "liba.a", "libb.a"] , ["libf.a", "libc.a", "libd.a", "libb.a", "liba.a"] , ["libc.a", "liba.a", "libb.a"] , ["libd.a", "libb.a", "liba.a"] ] } })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"( ["libg.a", "libe.a", "libf.a", "libc.a", "libd.a", "libb.a", "liba.a"] )"_json)); } SECTION("nub_left expression") { auto expr = Expression::FromJson(R"( {"type": "nub_left" , "$1": ["a", "b", "b", "a", "c", "b", "a"] })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["a", "b", "c"])"_json)); } SECTION("change_ending") { auto expr = Expression::FromJson(R"( { "type": "change_ending" , "$1": "PLACEHOLDER" , "ending": "_suffix" })"_json); REQUIRE(expr); expr = Replace(expr, "$1", ""s); REQUIRE(expr); auto empty_path = expr.Evaluate(env, fcts); REQUIRE(empty_path); REQUIRE(empty_path->IsString()); CHECK(empty_path == Expression::FromJson(R"("_suffix")"_json)); expr = Replace(expr, "$1", ".rc"s); REQUIRE(expr); auto hidden_file = expr.Evaluate(env, fcts); REQUIRE(hidden_file); REQUIRE(hidden_file->IsString()); CHECK(hidden_file == Expression::FromJson(R"(".rc_suffix")"_json)); expr = Replace(expr, "$1", "/root/path/file.txt"s); REQUIRE(expr); auto full_path = expr.Evaluate(env, fcts); REQUIRE(full_path); REQUIRE(full_path->IsString()); CHECK(full_path == Expression::FromJson(R"("/root/path/file_suffix")"_json)); } SECTION("basename") { auto expr = Expression::FromJson(R"( { "type": "basename" , "$1": "PLACEHOLDER" })"_json); REQUIRE(expr); expr = Replace(expr, "$1", "foo.c"s); REQUIRE(expr); auto plain_file = expr.Evaluate(env, fcts); REQUIRE(plain_file); REQUIRE(plain_file->IsString()); CHECK(plain_file == Expression::FromJson(R"("foo.c")"_json)); expr = Replace(expr, "$1", "/path/to/file.txt"s); REQUIRE(expr); auto stripped_path = expr.Evaluate(env, fcts); REQUIRE(stripped_path); REQUIRE(stripped_path->IsString()); CHECK(stripped_path == Expression::FromJson(R"("file.txt")"_json)); } SECTION("join") { auto expr = Expression::FromJson(R"( { "type": "join" , "$1": "PLACEHOLDER" , "separator": ";" })"_json); REQUIRE(expr); expr = Replace(expr, "$1", list_t{}); REQUIRE(expr); auto empty = expr.Evaluate(env, fcts); REQUIRE(empty); REQUIRE(empty->IsString()); CHECK(empty == Expression::FromJson(R"("")"_json)); expr = Replace(expr, "$1", list_t{foo}); REQUIRE(expr); auto single = expr.Evaluate(env, fcts); REQUIRE(single); REQUIRE(single->IsString()); CHECK(single == Expression::FromJson(R"("foo")"_json)); expr = Replace(expr, "$1", list_t{foo, bar, baz}); REQUIRE(expr); auto multi = expr.Evaluate(env, fcts); REQUIRE(multi); REQUIRE(multi->IsString()); CHECK(multi == Expression::FromJson(R"("foo;bar;baz")"_json)); // only list of strings are allowed expr = Replace(expr, "$1", foo); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "$1", list_t{foo, ExpressionPtr{number_t{}}}); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "$1", number_t{}); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("join_cmd expression") { auto expr = Expression::FromJson(R"( { "type": "join_cmd" , "$1": ["foo", "bar's", "baz"]})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("'foo' 'bar'\\''s' 'baz'")"_json)); expr = Expression::FromJson(R"( {"type": "join_cmd" , "$1": "not a list" } )"_json); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("escape_chars expression") { auto expr = Expression::FromJson(R"( { "type": "escape_chars" , "$1": "escape me X" , "chars": "abcX" , "escape_prefix": "X"})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("esXcXape me XX")"_json)); } SECTION("enumerate expression") { auto expr = Expression::FromJson(R"( { "type": "enumerate" , "$1": ["foo", "bar", "baz"] } )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson(R"( { "0000000000": "foo" , "0000000001": "bar" , "0000000002": "baz" } )"_json)); } SECTION("set expression") { auto expr = Expression::FromJson(R"( { "type": "set" , "$1": ["foo", "bar", "baz"] } )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson(R"( { "foo": true , "bar": true , "baz": true } )"_json)); } SECTION("reverse expression") { auto expr = Expression::FromJson(R"( { "type": "reverse" , "$1": ["foo", "bar", "baz"] } )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson(R"( ["baz", "bar", "foo"] )"_json)); } SECTION("length expression") { auto expr = Expression::FromJson(R"( { "type": "length" , "$1": ["foo", "bar", "baz"] } )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson("3"_json)); } SECTION("keys expression") { auto expr = Expression::FromJson(R"( { "type": "keys" , "$1": { "type": "'" , "$1": { "foo": true , "bar": false , "baz": true }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["bar", "baz", "foo"])"_json)); } SECTION("values expression") { auto expr = Expression::FromJson(R"( { "type": "values" , "$1": { "type": "'" , "$1": { "foo": true , "bar": "foo" , "baz": 1 }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foo", 1, true])"_json)); } SECTION("lookup expression") { auto expr = Expression::FromJson(R"( { "type": "lookup" , "key": "PLACEHOLDER" , "map": { "type": "'" , "$1": { "foo": true , "bar": 1 }}})"_json); REQUIRE(expr); expr = Replace(expr, "key", "foo"s); REQUIRE(expr); auto result_foo = expr.Evaluate(env, fcts); REQUIRE(result_foo); CHECK(result_foo == Expression::FromJson("true"_json)); expr = Replace(expr, "key", "bar"s); REQUIRE(expr); auto result_bar = expr.Evaluate(env, fcts); REQUIRE(result_bar); CHECK(result_bar == Expression::FromJson("1"_json)); // key baz is missing expr = Replace(expr, "key", "baz"s); REQUIRE(expr); auto result_baz = expr.Evaluate(env, fcts); REQUIRE(result_baz); CHECK(result_baz == Expression::FromJson("null"_json)); // map is not mapping expr = Replace(expr, "map", list_t{}); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // key is not string expr = Replace(expr, "key", number_t{}); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("lookup with default") { auto expr = Expression::FromJson(R"( { "type": "lookup" , "key": "PLACEHOLDER" , "map": { "type": "'" , "$1": { "foo": false , "bar": 1 , "baz" : null}} , "default" : { "type" : "join" , "separator": "x" , "$1": ["a", "b"]}})"_json); REQUIRE(expr); // Key present (and false) expr = Replace(expr, "key", "foo"s); REQUIRE(expr); auto result_foo = expr.Evaluate(env, fcts); REQUIRE(result_foo); CHECK(result_foo == Expression::FromJson("false"_json)); // Key present but value is null expr = Replace(expr, "key", "baz"s); REQUIRE(expr); auto result_baz = expr.Evaluate(env, fcts); REQUIRE(result_baz); CHECK(result_baz == Expression::FromJson(R"("axb")"_json)); // Key not present expr = Replace(expr, "key", "missing"s); REQUIRE(expr); auto result_missing = expr.Evaluate(env, fcts); REQUIRE(result_missing); CHECK(result_missing == Expression::FromJson(R"("axb")"_json)); } SECTION("array index") { auto expr = Expression::FromJson(R"( { "type": "[]" , "list": ["a", 101, "c", null, "e"] , "index": "PLACEHOLDER" , "default": "here be dragons" })"_json); REQUIRE(expr); // Index a number expr = Replace(expr, "index", Expression::FromJson("2"_json)); REQUIRE(expr); auto num_result = expr.Evaluate(env, fcts); REQUIRE(num_result); CHECK(num_result == Expression::FromJson(R"("c")"_json)); // Index a string expr = Replace(expr, "index", Expression::FromJson(R"("2")"_json)); REQUIRE(expr); auto string_result = expr.Evaluate(env, fcts); REQUIRE(string_result); CHECK(string_result == Expression::FromJson(R"("c")"_json)); // Index pointing to a null value expr = Replace(expr, "index", Expression::FromJson("3"_json)); REQUIRE(expr); auto null_result = expr.Evaluate(env, fcts); REQUIRE(null_result); CHECK(null_result == Expression::FromJson("null"_json)); // Index out of range expr = Replace(expr, "index", Expression::FromJson("5"_json)); REQUIRE(expr); auto default_result = expr.Evaluate(env, fcts); REQUIRE(default_result); CHECK(default_result == Expression::FromJson(R"("here be dragons")"_json)); // Negative index, number expr = Replace(expr, "index", Expression::FromJson("-3"_json)); REQUIRE(expr); auto neg_index_number = expr.Evaluate(env, fcts); REQUIRE(neg_index_number); CHECK(neg_index_number == Expression::FromJson(R"("c")"_json)); // Negative index, extreme number expr = Replace(expr, "index", Expression::FromJson("-5"_json)); REQUIRE(expr); auto neg_index_number_extreme = expr.Evaluate(env, fcts); REQUIRE(neg_index_number_extreme); CHECK(neg_index_number_extreme == Expression::FromJson(R"("a")"_json)); // Negative index, string expr = Replace(expr, "index", Expression::FromJson(R"("-3")"_json)); REQUIRE(expr); auto neg_index_string = expr.Evaluate(env, fcts); REQUIRE(neg_index_string); CHECK(neg_index_string == Expression::FromJson(R"("c")"_json)); // Index out of range, other direction expr = Replace(expr, "index", Expression::FromJson("-6"_json)); REQUIRE(expr); auto other_default_result = expr.Evaluate(env, fcts); REQUIRE(other_default_result); CHECK(other_default_result == Expression::FromJson(R"("here be dragons")"_json)); } SECTION("empty_map expression") { auto expr = Expression::FromJson(R"({"type": "empty_map"})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson("{}"_json)); } SECTION("singleton_map expression") { auto expr = Expression::FromJson(R"( { "type": "singleton_map" , "key": "foo" , "value": "bar"})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo": "bar"})"_json)); } SECTION("disjoint_map_union expression") { auto expr = Expression::FromJson(R"( { "type": "disjoint_map_union" , "$1": "PLACEHOLDER" })"_json); REQUIRE(expr); auto literal_foo = Expression::FromJson(R"({"type": "'", "$1": {"foo":true}})"_json); REQUIRE(literal_foo); auto literal_foo_false = Expression::FromJson(R"({"type": "'", "$1": {"foo":false}})"_json); REQUIRE(literal_foo_false); auto literal_bar = Expression::FromJson(R"({"type": "'", "$1": {"bar":false}})"_json); REQUIRE(literal_bar); expr = Replace(expr, "$1", list_t{literal_foo, literal_bar}); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo": true, "bar": false})"_json)); // duplicate foo, but with same value expr = Replace(expr, "$1", list_t{literal_foo, literal_foo}); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo": true})"_json)); // duplicate foo, but with different value expr = Replace(expr, "$1", list_t{literal_foo, literal_foo_false}); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // empty list should produce empty map expr = Replace(expr, "$1", list_t{}); REQUIRE(expr); auto empty = expr.Evaluate(env, fcts); REQUIRE(empty); REQUIRE(empty->IsMap()); REQUIRE(empty == Expression::FromJson("{}"_json)); } SECTION("map_union expression") { auto expr = Expression::FromJson(R"( { "type": "map_union" , "$1": { "type": "'" , "$1": [ {"foo": true} , {"bar": false}] }})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo": true, "bar": false})"_json)); // empty list should produce empty map expr = Expression::FromJson(R"({"type": "map_union", "$1": []})"_json); REQUIRE(expr); auto empty = expr.Evaluate(env, fcts); REQUIRE(empty); REQUIRE(empty->IsMap()); REQUIRE(empty == Expression::FromJson("{}"_json)); } SECTION("to_subdir expression") { auto expr = Expression::FromJson(R"( { "type": "to_subdir" , "subdir": "prefix" , "$1": { "type": "'" , "$1": { "foo": "hello" , "bar": "world" }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson( R"({"prefix/foo": "hello", "prefix/bar": "world"})"_json)); } SECTION("to_subdir expression with conflict") { auto expr = Expression::FromJson(R"( { "type": "to_subdir" , "subdir": "prefix" , "$1": { "type": "'" , "$1": { "foo": "hello" , "./foo": "world" }}})"_json); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("flat to_subdir without proper conflict") { auto expr = Expression::FromJson(R"( { "type": "to_subdir" , "subdir": "prefix" , "flat" : "YES" , "$1": { "type": "'" , "$1": { "foobar/data/foo": "hello" , "foobar/include/foo": "hello" , "bar": "world" }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson( R"({"prefix/foo": "hello", "prefix/bar": "world"})"_json)); } SECTION("flat to_subdir with conflict") { auto expr = Expression::FromJson(R"( { "type": "to_subdir" , "subdir": "prefix" , "flat" : "YES" , "$1": { "type": "'" , "$1": { "foobar/data/foo": "HELLO" , "foobar/include/foo": "hello" , "bar": "world" }}})"_json); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("from_subdir") { auto expr = Expression::FromJson(R"( {"type": "from_subdir", "subdir": "foo" , "$1": {"type": "'", "$1": { "foo/a/b/c": "abc.txt" , "foo/a/other": "other.txt" , "foo/top": "top.xt" , "foo/a/b/../d/e": "make canonical" , "bar/a/b/c": "ignore bar/a/b/c" , "bar/a/b/../b/c": "also ingnore other path" }}} )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson(R"( { "a/b/c": "abc.txt" , "a/other": "other.txt" , "top": "top.xt" , "a/d/e": "make canonical" } )"_json)); } SECTION("from_subdir trivial conflict") { auto expr = Expression::FromJson(R"( {"type": "from_subdir", "subdir": "foo" , "$1": {"type": "'", "$1": { "foo/a/b/c": "abc.txt" , "foo/a/b/../b/c": "abc.txt" }}} )"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); CHECK(result == Expression::FromJson(R"( {"a/b/c": "abc.txt"} )"_json)); } SECTION("from_subdir conflict") { auto expr = Expression::FromJson(R"( {"type": "from_subdir", "subdir": "foo" , "$1": {"type": "'", "$1": { "foo/a/b/c": "one value" , "foo/a/b/../b/c": "different value" }}} )"_json); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } fcts = FunctionMap::MakePtr( fcts, "concat", [](auto&& eval, auto const& expr, auto const& env) { auto p1 = eval(expr->Get("$1", ""s), env); auto p2 = eval(expr->Get("$2", ""s), env); return ExpressionPtr{p1->String() + p2->String()}; }); SECTION("foreach expression") { auto expr = Expression::FromJson(R"( { "type": "foreach" , "var": "x" , "range": ["foo", "bar", "baz"] , "body": { "type": "concat" , "$1": { "type": "var" , "name": "x" } , "$2": "y" }})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["fooy", "bary", "bazy"])"_json)); } SECTION("foreach_map expression") { auto expr = Expression::FromJson(R"( { "type": "foreach_map" , "var_key": "key" , "var_val": "val" , "body": { "type": "concat" , "$1": { "type": "var" , "name": "key" } , "$2": { "type": "var" , "name": "val" }}})"_json); REQUIRE(expr); // range is missing (should default to empty map) auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"([])"_json)); // range is map with one entry expr = Add(expr, "range", Expression::FromJson(R"( { "type": "'" , "$1": {"foo": "bar"}})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foobar"])"_json)); // range is map with multiple entries expr = Replace(expr, "range", Expression::FromJson(R"( { "type": "'" , "$1": {"foo": "bar", "bar": "baz"}})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["barbaz", "foobar"])"_json)); // fail if range is string expr = Replace(expr, "range", Expression::FromJson(R"("foo")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if range is number expr = Replace(expr, "range", Expression::FromJson(R"("4711")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if range is Boolean expr = Replace(expr, "range", Expression::FromJson(R"("true")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("zip_with expression") { auto expr = Expression::FromJson(R"( { "type": "zip_with" , "var_1": "key" , "var_2": "val" , "body": { "type": "concat" , "$1": { "type": "var" , "name": "key" } , "$2": { "type": "var" , "name": "val" }}})"_json); REQUIRE(expr); // ranges are missing (should default to empty list) auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"([])"_json)); // one range is missing (should default to empty list) expr = Add(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"([])"_json)); // ranges are lists with one entry expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": ["foo"]})"_json)); expr = Add(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": ["bar"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foobar"])"_json)); // ranges are lists with multiple entries expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": ["foo", "bar"]})"_json)); expr = Replace(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": ["bar", "baz"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foobar", "barbaz"])"_json)); // ranges are lists with different sizes expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": ["foo", "bar"]})"_json)); expr = Replace(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": ["bar"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsList()); CHECK(result == Expression::FromJson(R"(["foobar"])"_json)); // fail if ranges are string expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_2", Expression::FromJson(R"("foo")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_1", Expression::FromJson(R"("foo")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if ranges are number expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_2", Expression::FromJson(R"("4711")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_1", Expression::FromJson(R"("4711")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if ranges are Boolean expr = Replace(expr, "range_1", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_2", Expression::FromJson(R"("true")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_2", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_1", Expression::FromJson(R"("true")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("zip_map expression") { auto expr = Expression::FromJson(R"( { "type": "zip_map" })"_json); REQUIRE(expr); // ranges are missing (should default to empty map) auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({})"_json)); // one range is missing (should default to empty map) expr = Add(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({})"_json)); // ranges are lists with one entry expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": ["foo"]})"_json)); expr = Add(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": ["bar"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo" : "bar"})"_json)); // ranges are lists with multiple entries expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": ["foo", "bar"]})"_json)); expr = Replace(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": ["bar", "baz"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"bar": "baz", "foo": "bar"})"_json)); // ranges are lists with different sizes expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": ["foo", "bar"]})"_json)); expr = Replace(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": ["bar"]})"_json)); REQUIRE(expr); result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson(R"({"foo": "bar"})"_json)); // fail if ranges are string expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_val", Expression::FromJson(R"("foo")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_key", Expression::FromJson(R"("foo")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if ranges are number expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_val", Expression::FromJson(R"("4711")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_key", Expression::FromJson(R"("4711")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); // fail if ranges are Boolean expr = Replace(expr, "range_key", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_val", Expression::FromJson(R"("true")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); expr = Replace(expr, "range_val", Expression::FromJson(R"( { "type": "'" , "$1": [""]})"_json)); expr = Replace(expr, "range_key", Expression::FromJson(R"("true")"_json)); REQUIRE(expr); CHECK_FALSE(expr.Evaluate(env, fcts)); } SECTION("foldl expression") { auto expr = Expression::FromJson(R"( { "type": "foldl" , "var": "x" , "range": ["bar", "baz"] , "accum_var": "a" , "start": "foo" , "body": { "type": "concat" , "$1": { "type": "var" , "name": "x" } , "$2": { "type": "var" , "name": "a" }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("bazbarfoo")"_json)); } SECTION("let* expression") { auto expr = Expression::FromJson(R"( { "type": "let*" , "bindings": [ ["foo", "foo"] , ["bar", "bar"] ] , "body": { "type": "concat" , "$1": { "type": "var" , "name": "foo" } , "$2": { "type": "var" , "name": "bar" }}})"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("foobar")"_json)); } SECTION("sequentiallity of let* expression") { auto expr = Expression::FromJson(R"( { "type": "let*" , "bindings": [ ["one", "foo"] , ["two", { "type": "join" , "$1": [ {"type": "var", "name" : "one"} , {"type": "var", "name" : "one"} ]}] , ["four", { "type": "join" , "$1": [ {"type": "var", "name" : "two"} , {"type": "var", "name" : "two"} ]}] ] , "body": { "type" : "var" , "name" : "four" } })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsString()); CHECK(result == Expression::FromJson(R"("foofoofoofoo")"_json)); } SECTION("env expression") { auto env = Configuration{Expression::FromJson( R"({"foo": "FOO_STRING", "bar": "BAR_STRING"})"_json)}; auto expr = Expression::FromJson(R"( { "type": "env" , "vars": ["bar", "baz"] })"_json); REQUIRE(expr); auto result = expr.Evaluate(env, fcts); REQUIRE(result); REQUIRE(result->IsMap()); CHECK(result == Expression::FromJson( R"({"bar": "BAR_STRING", "baz": null})"_json)); auto empty = Expression::FromJson(R"({"type": "env"})"_json); REQUIRE(empty); auto none = empty.Evaluate(env, fcts); REQUIRE(none); REQUIRE(none->IsMap()); CHECK(none == Expression::kEmptyMap); } SECTION("concat_target_name expression") { auto expr = Expression::FromJson(R"( { "type": "concat_target_name" , "$1": "PLACEHOLDER" , "$2": "_suffix" })"_json); REQUIRE(expr); expr = Replace(expr, "$1", "foo"s); REQUIRE(expr); auto str_result = expr.Evaluate(env, fcts); REQUIRE(str_result); REQUIRE(str_result->IsString()); CHECK(str_result == Expression::FromJson(R"("foo_suffix")"_json)); auto dep_tgt = Expression::FromJson(R"(["subdir", "bar"])"_json); REQUIRE(dep_tgt); expr = Replace(expr, "$1", dep_tgt); REQUIRE(expr); auto dep_result = expr.Evaluate(env, fcts); REQUIRE(dep_result); REQUIRE(dep_result->IsList()); CHECK(dep_result == Expression::FromJson(R"(["subdir", "bar_suffix"])"_json)); } SECTION("range expression") { auto expr_str = Expression::FromJson(R"( { "type": "range" , "$1": "3" })"_json); REQUIRE(expr_str); auto str_result = expr_str.Evaluate(env, fcts); CHECK(str_result == Expression::FromJson(R"(["0", "1", "2"])"_json)); auto expr_number = Expression::FromJson(R"( { "type": "range" , "$1": 4 })"_json); REQUIRE(expr_number); auto number_result = expr_number.Evaluate(env, fcts); CHECK(number_result == Expression::FromJson(R"(["0", "1", "2", "3"])"_json)); auto expr_null = Expression::FromJson(R"( { "type": "range" , "$1": null })"_json); REQUIRE(expr_null); auto null_result = expr_null.Evaluate(env, fcts); CHECK(null_result == Expression::FromJson(R"([])"_json)); } } TEST_CASE("Expression Assertions", "[expression]") { using namespace std::string_literals; auto env = Configuration{}; auto fcts = FunctionMapPtr{}; SECTION("fail") { auto expr = Expression::FromJson(R"( { "type": "fail" , "msg": {"type": "join", "$1": ["ErRoR", "mEsSaGe"]} } )"_json); REQUIRE(expr); std::stringstream log{}; CHECK(not expr.Evaluate(env, fcts, [&](auto msg) { log << msg; })); CHECK(log.str().find("ErRoRmEsSaGe") != std::string::npos); } SECTION("assert_non_empty") { auto expr = Expression::FromJson(R"( { "type": "assert_non_empty" , "msg": "Found-Empty!!" , "$1": {"type": "var", "name": "x"} } )"_json); REQUIRE(expr); auto list = Expression::FromJson(R"([1, 2, 3])"_json); CHECK(expr.Evaluate(env.Update("x", list), fcts) == list); auto map = Expression::FromJson(R"({"foo": "bar"})"_json); CHECK(expr.Evaluate(env.Update("x", map), fcts) == map); auto empty_list = Expression::FromJson(R"([])"_json); std::stringstream log_list{}; CHECK(not expr.Evaluate(env.Update("x", empty_list), fcts, [&](auto msg) { log_list << msg; })); CHECK(log_list.str().find("Found-Empty!!") != std::string::npos); auto empty_map = Expression::FromJson(R"({})"_json); std::stringstream log_map{}; CHECK(not expr.Evaluate(env.Update("x", empty_map), fcts, [&](auto msg) { log_map << msg; })); CHECK(log_map.str().find("Found-Empty!!") != std::string::npos); } SECTION("assert") { auto expr = Expression::FromJson(R"( { "type": "assert" , "predicate": {"type": "[]", "index": 0 , "list": {"type": "var", "name": "_"}} , "msg": ["First entry UNTRUE", {"type": "var", "name": "_"}] , "$1": {"type": "++", "$1": [{"type": "var", "name": "x"} , ["b", "c"]]} })"_json); REQUIRE(expr); CHECK(expr.Evaluate( env.Update("x", Expression::FromJson(R"(["a"])"_json)), fcts) == Expression::FromJson(R"(["a", "b", "c"])"_json)); std::stringstream log{}; CHECK(not expr.Evaluate( env.Update("x", Expression::FromJson(R"([false, "foo"])"_json)), fcts, [&](auto msg) { log << msg; })); // log must contain the canoncial (minimal) repesentation of evaluating // "msg" CHECK( log.str().find(R"(["First entry UNTRUE",[false,"foo","b","c"])"s) != std::string::npos); } } TEST_CASE("Expression hash computation", "[expression]") { using namespace std::string_literals; using path = std::filesystem::path; using number_t = Expression::number_t; using result_t = Expression::result_t; using list_t = Expression::list_t; using map_t = Expression::map_t; auto none = ExpressionPtr{}; auto boolean = ExpressionPtr{false}; auto number = ExpressionPtr{number_t{}}; auto string = ExpressionPtr{""s}; auto artifact = ExpressionPtr{ArtifactDescription::CreateTree(path{""})}; auto result = ExpressionPtr{result_t{}}; auto list = ExpressionPtr{list_t{}}; auto map = ExpressionPtr{map_t{}}; CHECK_FALSE(none->ToHash().empty()); CHECK(none->ToHash() == Expression{}.ToHash()); CHECK_FALSE(boolean->ToHash().empty()); CHECK(boolean->ToHash() == Expression{false}.ToHash()); CHECK_FALSE(boolean->ToHash() == Expression{true}.ToHash()); CHECK_FALSE(number->ToHash().empty()); CHECK(number->ToHash() == Expression{number_t{}}.ToHash()); CHECK_FALSE(number->ToHash() == Expression{number_t{1}}.ToHash()); CHECK_FALSE(string->ToHash().empty()); CHECK(string->ToHash() == Expression{""s}.ToHash()); CHECK_FALSE(string->ToHash() == Expression{" "s}.ToHash()); CHECK_FALSE(artifact->ToHash().empty()); CHECK(artifact->ToHash() == Expression{ArtifactDescription::CreateTree(path{""})}.ToHash()); CHECK_FALSE( artifact->ToHash() == Expression{ArtifactDescription::CreateTree(path{" "})}.ToHash()); CHECK_FALSE(result->ToHash().empty()); CHECK(result->ToHash() == Expression{result_t{}}.ToHash()); CHECK_FALSE( result->ToHash() == Expression{ result_t{.artifact_stage = boolean, .provides = {}, .runfiles = {}}} .ToHash()); CHECK_FALSE(list->ToHash().empty()); CHECK(list->ToHash() == Expression{list_t{}}.ToHash()); CHECK_FALSE(list->ToHash() == Expression{list_t{number}}.ToHash()); CHECK_FALSE(list->ToHash() == Expression{map_t{{""s, number}}}.ToHash()); CHECK_FALSE(map->ToHash().empty()); CHECK(map->ToHash() == Expression{map_t{}}.ToHash()); CHECK_FALSE(map->ToHash() == Expression{map_t{{""s, number}}}.ToHash()); CHECK_FALSE(map->ToHash() == Expression{list_t{string, number}}.ToHash()); auto exprs = std::vector{ none, boolean, number, string, artifact, result, list, map}; for (auto const& l : exprs) { for (auto const& r : exprs) { if (&l != &r) { CHECK_FALSE(l->ToHash() == r->ToHash()); } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/expression/linked_map.test.cpp000066400000000000000000000206221516554100600332050ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/expression/linked_map.hpp" #include #include #include #include #include #include #include // std::move #include #include "catch2/catch_test_macros.hpp" TEST_CASE("Empty map", "[linked_map]") { using map_t = LinkedMap; auto map = map_t::MakePtr(map_t::underlying_map_t{}); REQUIRE(map); CHECK(map->empty()); auto empty_map = map_t::underlying_map_t{}; map = map_t::MakePtr(map, empty_map); REQUIRE(map); CHECK(map->empty()); auto empty_linked_map = map_t::MakePtr(empty_map); map = map_t::MakePtr(map, empty_linked_map); REQUIRE(map); CHECK(map->empty()); } TEST_CASE("Lookup and iteration", "[linked_map]") { using map_t = LinkedMap; constexpr int kCount{100}; constexpr int kQ{10}; // kQ == gcd(kCount, kQ) && 0 < kCount / kQ < 10 auto map = map_t::MakePtr("0", 0); REQUIRE(map); CHECK(not map->empty()); CHECK(map->size() == 1); for (int i{1}; i < kCount; ++i) { auto update = map_t::underlying_map_t{{std::to_string(i / kQ), i}}; if (i % 2 == 0) { // update via underlying map map = map_t::MakePtr(map, update); } else { // update via linked map ptr map = map_t::MakePtr(map, map_t::MakePtr(update)); } REQUIRE(map); CHECK(map->size() == static_cast((i / kQ) + 1)); } SECTION("contains and lookup") { for (int i{0}; i < kCount / kQ; ++i) { auto key = std::to_string(i); // kQ-many values per key: i -> i*kQ + [0;kQ-1], expect last auto expect = i * kQ + (kQ - 1); CHECK(map->contains(key)); CHECK(map->at(key) == expect); } } SECTION("iteration via ranged-based loop") { auto i = kQ - 1; for (auto const& el : *map) { CHECK(el.first == std::to_string(i / kQ)); CHECK(el.second == i); i += kQ; } } SECTION("iteration via algorithm") { auto i = kQ - 1; std::for_each(std::begin(*map), std::end(*map), [&](auto const& el) { CHECK(el.first == std::to_string(i / kQ)); CHECK(el.second == i); i += kQ; }); } } class CopyCounter { public: CopyCounter() : count_{std::make_shared()} {} CopyCounter(CopyCounter const& other) : count_{other.count_} { ++(*other.count_); } CopyCounter(CopyCounter&&) = default; ~CopyCounter() = default; auto operator=(CopyCounter const& other) -> CopyCounter& { if (this != &other) { ++(*other.count_); count_ = other.count_; } return *this; } auto operator=(CopyCounter&&) -> CopyCounter& = default; [[nodiscard]] auto Count() const -> std::size_t { return *count_; } private: // all copies of this object share the same counter std::shared_ptr count_; }; TEST_CASE("Zero copies", "[linked_map]") { using map_t = LinkedMap; constexpr int kCount{100}; auto map = map_t::Ptr{}; SECTION("Via initializer list") { for (int i{0}; i < kCount; ++i) { map = map_t::MakePtr(map, {{std::to_string(i), CopyCounter{}}}); REQUIRE(map); } for (int i{0}; i < kCount; ++i) { auto key = std::to_string(i); REQUIRE(map->contains(key)); // underlying map's initializer_list produces a single copy CHECK(map->at(key).Count() == 1); } } SECTION("Via pair") { for (int i{0}; i < kCount; ++i) { map = map_t::MakePtr(map, {std::to_string(i), CopyCounter{}}); REQUIRE(map); } for (int i{0}; i < kCount; ++i) { auto key = std::to_string(i); REQUIRE(map->contains(key)); CHECK(map->at(key).Count() == 0); } } SECTION("Via key and value arguments") { for (int i{0}; i < kCount; ++i) { map = map_t::MakePtr(map, std::to_string(i), CopyCounter{}); REQUIRE(map); } for (int i{0}; i < kCount; ++i) { auto key = std::to_string(i); REQUIRE(map->contains(key)); CHECK(map->at(key).Count() == 0); } } SECTION("Via underlying map and emplace") { for (int i{0}; i < kCount; ++i) { map_t::underlying_map_t update{}; update.emplace(std::to_string(i), CopyCounter()); map = map_t::MakePtr(map, std::move(update)); REQUIRE(map); } for (int i{0}; i < kCount; ++i) { auto key = std::to_string(i); REQUIRE(map->contains(key)); CHECK(map->at(key).Count() == 0); } } SECTION("Via linked map ptr") { for (int i{0}; i < kCount; ++i) { auto update = map_t::MakePtr(std::to_string(i), CopyCounter{}); map = map_t::MakePtr(map, std::move(update)); REQUIRE(map); } for (int i{0}; i < kCount; ++i) { auto key = std::to_string(i); REQUIRE(map->contains(key)); CHECK(map->at(key).Count() == 0); } } } // Custom container that holds a LinkedMap. class CustomContainer { public: class Ptr; using linked_map_t = LinkedMap; // Special smart pointer for container that can be used as internal NextPtr // for LinkedMap by implementing IsNotNull(), LinkedMap(), and Make(). class Ptr : public std::shared_ptr { public: [[nodiscard]] auto IsNotNull() const noexcept -> bool { return static_cast(*this); } [[nodiscard]] auto Map() const& -> linked_map_t const& { return (*this)->Map(); } [[nodiscard]] static auto Make(linked_map_t&& map) -> Ptr { return Ptr{std::make_shared(std::move(map))}; } }; explicit CustomContainer(linked_map_t&& map) noexcept : map_{std::move(map)} {} [[nodiscard]] auto Map() & noexcept -> linked_map_t& { return map_; } private: linked_map_t map_; }; TEST_CASE("Custom NextPtr", "[linked_map]") { using map_t = LinkedMap; constexpr int kCount{100}; constexpr int kQ{10}; auto container = CustomContainer::Ptr::Make(map_t{0, 0}); REQUIRE(container); CHECK(container->Map().size() == 1); for (int i{1}; i < kCount; ++i) { container = CustomContainer::Ptr::Make(map_t{container, {{i / kQ, i}}}); REQUIRE(container); CHECK(container->Map().size() == static_cast(i / kQ + 1)); } for (int i{0}; i < kCount / kQ; ++i) { auto key = i; // kQ-many values per key: i -> i*kQ + [0;kQ-1], expect last auto expect = i * kQ + (kQ - 1); CHECK(container->Map().contains(key)); CHECK(container->Map().at(key) == expect); } } TEST_CASE("Hash computation", "[linked_map]") { using map_t = LinkedMap; auto map = map_t::MakePtr("foo", 4711); // NOLINT REQUIRE(map); CHECK(not map->empty()); auto map_hash = std::hash>{}(*map); CHECK_FALSE(map_hash == 0); auto ptr_hash = std::hash>{}(map); CHECK_FALSE(ptr_hash == 0); CHECK(ptr_hash == map_hash); map = map_t::MakePtr(map, "foo", 4711); // NOLINT auto dup_hash = std::hash>{}(map); CHECK_FALSE(dup_hash == 0); CHECK(dup_hash == map_hash); map = map_t::MakePtr(map, "bar", 4712); // NOLINT auto upd_hash = std::hash>{}(map); CHECK_FALSE(upd_hash == 0); CHECK_FALSE(upd_hash == map_hash); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/000077500000000000000000000000001516554100600273425ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/TARGETS000066400000000000000000000122451516554100600304020ustar00rootroot00000000000000{ "result_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["result_map"] , "srcs": ["result_map.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , [ "@" , "src" , "src/buildtool/build_engine/analysed_target" , "graph_information" ] , ["@", "src", "src/buildtool/build_engine/analysed_target", "target"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , ["@", "src", "src/buildtool/build_engine/target_map", "result_map"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common", "tree"] , ["@", "src", "src/buildtool/common", "tree_overlay"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "target_map"] } , "target_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["target_map"] , "srcs": ["target_map.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/build_engine/analysed_target", "target"] , ["@", "src", "src/buildtool/build_engine/base_maps", "directory_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "entity_name_data"] , ["@", "src", "src/buildtool/build_engine/base_maps", "expression_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "rule_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "source_map"] , ["@", "src", "src/buildtool/build_engine/base_maps", "targets_file_map"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , [ "@" , "src" , "src/buildtool/build_engine/target_map" , "absent_target_map" ] , [ "@" , "src" , "src/buildtool/build_engine/target_map" , "configured_target" ] , ["@", "src", "src/buildtool/build_engine/target_map", "result_map"] , ["@", "src", "src/buildtool/build_engine/target_map", "target_map"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common", "tree"] , ["@", "src", "src/buildtool/common", "tree_overlay"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "api_bundle"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/execution_api/remote", "context"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/main", "analyse_context"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/serve_api/remote", "serve_api"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_serve_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "build_engine", "target_map"] } , "target_map_internals": { "type": ["@", "rules", "CC/test", "test"] , "name": ["target_map_internals"] , "srcs": ["target_map_internals.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , [ "@" , "src" , "src/buildtool/build_engine/target_map" , "target_map_testable_internals" ] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "build_engine", "target_map"] } , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [ ["TREE", null, "data_src"] , ["TREE", null, "data_targets"] , ["TREE", null, "data_rules"] ] , "stage": ["test", "buildtool", "build_engine", "target_map"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["target_map"] , "deps": ["result_map", "target_map", "target_map_internals"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/000077500000000000000000000000001516554100600314655ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/result/000077500000000000000000000000001516554100600330035ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/result/RULES000066400000000000000000000046221516554100600336240ustar00rootroot00000000000000{ "wrong RESULT": { "string_fields": ["runfiles", "runfiles_entry", "artifacts", "artifacts_entry", "provides"] , "expression": { "type": "let*" , "bindings": [["artifact", {"type": "BLOB"}]] , "body": { "type": "RESULT" , "artifacts": { "type": "if" , "cond": {"type": "FIELD", "name": "artifacts"} , "then": [ { "type": "join" , "$1": ["artifacts", "not", "a", "map"] , "separator": "-" } ] , "else": { "type": "if" , "cond": {"type": "FIELD", "name": "artifacts_entry"} , "then": { "type": "singleton_map" , "key": { "type": "join" , "$1": ["bad", "artifact", "path"] , "separator": "-" } , "value": { "type": "join" , "$1": ["bad", "artifact", "entry"] , "separator": "-" } } , "else": { "type": "singleton_map" , "key": "OK" , "value": {"type": "var", "name": "artifact"} } } } , "runfiles": { "type": "if" , "cond": {"type": "FIELD", "name": "runfiles"} , "then": [ { "type": "join" , "$1": ["runfiles", "not", "a", "map"] , "separator": "-" } ] , "else": { "type": "if" , "cond": {"type": "FIELD", "name": "runfiles_entry"} , "then": { "type": "singleton_map" , "key": { "type": "join" , "$1": ["bad", "runfiles", "path"] , "separator": "-" } , "value": { "type": "join" , "$1": ["bad", "runfiles", "entry"] , "separator": "-" } } , "else": { "type": "singleton_map" , "key": "OK" , "value": {"type": "var", "name": "artifact"} } } } , "provides": { "type": "if" , "cond": {"type": "FIELD", "name": "provides"} , "then": [ { "type": "join" , "$1": ["provides", "not", "a", "map"] , "separator": "-" } ] , "else": {"type": "singleton_map", "key": "OK", "value": "OK value"} } } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/rule/000077500000000000000000000000001516554100600324345ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/rule/RULES000066400000000000000000000000031516554100600332420ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/simple_rules/000077500000000000000000000000001516554100600341705ustar00rootroot00000000000000RULES000066400000000000000000000130021516554100600347220ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/simple_rules{ "just provide": { "expression": { "type": "RESULT" , "provides": {"type": "singleton_map", "key": "foo", "value": "bar"} } } , "provide variable FOO": { "config_vars": ["FOO"] , "expression": { "type": "RESULT" , "provides": { "type": "singleton_map" , "key": "foo" , "value": {"type": "var", "name": "FOO"} } } } , "transition FOO": { "config_fields": ["value"] , "target_fields": ["deps"] , "config_transitions": { "deps": [ { "type": "singleton_map" , "key": "FOO" , "value": {"type": "join", "$1": {"type": "FIELD", "name": "value"}} } ] } , "expression": { "type": "RESULT" , "provides": { "type": "singleton_map" , "key": "transitioned deps" , "value": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "deps"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "x"} , "transition": { "type": "singleton_map" , "key": "FOO" , "value": {"type": "join", "$1": {"type": "FIELD", "name": "value"}} } , "provider": "foo" } } } } } , "collect deps": { "target_fields": ["deps"] , "expression": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } } } , "collect deps as runfiles": { "target_fields": ["deps"] , "expression": { "type": "RESULT" , "runfiles": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } } } , "text file": { "string_fields": ["name", "content"] , "target_fields": ["analyze"] , "expression": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} , "value": { "type": "BLOB" , "data": {"type": "join", "$1": {"type": "FIELD", "name": "content"}} } } } } , "symlink": { "string_fields": ["name", "content"] , "target_fields": ["analyze"] , "expression": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} , "value": { "type": "SYMLINK" , "data": {"type": "join", "$1": {"type": "FIELD", "name": "content"}} } } } } , "implicit file": { "implicit": {"script": ["implicit_script.sh"]} , "expression": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "script"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } } } , "upper case": { "target_fields": ["srcs"] , "expression": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "input" , "range": {"type": "FIELD", "name": "srcs"} , "body": { "type": "disjoint_map_union" , "$1": { "type": "let*" , "bindings": [ [ "input_artifacts" , { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "input"} } ] , [ "names" , { "type": "keys" , "$1": {"type": "var", "name": "input_artifacts"} } ] ] , "body": { "type": "foreach" , "var": "x" , "range": {"type": "var", "name": "names"} , "body": { "type": "let*" , "bindings": [ [ "upper" , { "type": "ACTION" , "inputs": { "type": "singleton_map" , "key": "in" , "value": { "type": "lookup" , "map": {"type": "var", "name": "input_artifacts"} , "key": {"type": "var", "name": "x"} } } , "outs": ["out"] , "cmd": ["/bin/sh", "-c", "tr 'a-z' 'A-Z' < in > out"] } ] ] , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "x"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "upper"} , "key": "out" } } } } } } } } } } , "action": { "string_fields": ["cmd", "outs", "out_dirs"] , "expression": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "cmd": {"type": "FIELD", "name": "cmd"} , "outs": {"type": "FIELD", "name": "outs"} , "out_dirs": {"type": "FIELD", "name": "out_dirs"} } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/tree/000077500000000000000000000000001516554100600324245ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/tree/RULES000066400000000000000000000025541516554100600332470ustar00rootroot00000000000000{ "ls -R": { "target_fields": ["tree", "direct"] , "expression": { "type": "let*" , "bindings": [ [ "tree" , { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "tree"} , "body": {"type": "DEP_RUNFILES", "dep": {"type": "var", "name": "dep"}} } } } ] , [ "direct" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "direct"} , "body": {"type": "DEP_RUNFILES", "dep": {"type": "var", "name": "dep"}} } } ] , [ "inputs" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "tree" , "value": {"type": "var", "name": "tree"} } , {"type": "var", "name": "direct"} ] } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "outs": ["_out"] , "inputs": {"type": "var", "name": "inputs"} , "cmd": ["sh", "-c", "find . -name '*.txt' > _out"] } } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/tree_overlay/000077500000000000000000000000001516554100600341655ustar00rootroot00000000000000RULES000066400000000000000000000024511516554100600347250ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_rules/tree_overlay{ "overlay": { "target_fields": ["deps"] , "expression": { "type": "let*" , "bindings": [ [ "deps" , { "type": "foreach" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} } ] , [ "overlay tree" , {"type": "TREE_OVERLAY", "$1": {"type": "var", "name": "deps"}} ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": "it" , "value": {"type": "var", "name": "overlay tree"} } } } } , "disjoint overlay": { "target_fields": ["deps"] , "expression": { "type": "let*" , "bindings": [ [ "deps" , { "type": "foreach" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} } ] , [ "overlay tree" , { "type": "DISJOINT_TREE_OVERLAY" , "$1": {"type": "var", "name": "deps"} } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": "it" , "value": {"type": "var", "name": "overlay tree"} } } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/000077500000000000000000000000001516554100600311225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/000077500000000000000000000000001516554100600313425ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/b/000077500000000000000000000000001516554100600315635ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/b/targets_here/000077500000000000000000000000001516554100600342375ustar00rootroot00000000000000c/000077500000000000000000000000001516554100600344025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/b/targets_hered/000077500000000000000000000000001516554100600346255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/b/targets_here/cfoo000066400000000000000000000000041516554100600353250ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/a/b/targets_here/c/dfoo just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/file_reference/000077500000000000000000000000001516554100600340575ustar00rootroot00000000000000hello.txt000066400000000000000000000000141516554100600356370ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/file_referencehello world just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/foo000066400000000000000000000000001516554100600316160ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_rules/000077500000000000000000000000001516554100600336255ustar00rootroot00000000000000implicit_script.sh000066400000000000000000000011741516554100600373030ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_rules#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. echo Hello World just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_targets/000077500000000000000000000000001516554100600341445ustar00rootroot00000000000000bar.txt000066400000000000000000000000041516554100600353640ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_targetsbar baz.txt000066400000000000000000000000041516554100600353740ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_targetsbaz foo.txt000066400000000000000000000000041516554100600354030ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/simple_targetsfoo just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree/000077500000000000000000000000001516554100600320615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree/foo.txt000066400000000000000000000000001516554100600333730ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree/tree/000077500000000000000000000000001516554100600330205ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree/tree/foo.txt000066400000000000000000000000001516554100600343320ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree_overlay/000077500000000000000000000000001516554100600336225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/tree_overlay/x000066400000000000000000000000001516554100600340020ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/000077500000000000000000000000001516554100600313715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/foo000066400000000000000000000000001516554100600320650ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/000077500000000000000000000000001516554100600316405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/foo000066400000000000000000000000001516554100600323340ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/000077500000000000000000000000001516554100600321075ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/foo000066400000000000000000000000001516554100600326030ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/x/000077500000000000000000000000001516554100600323565ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/x/foo000066400000000000000000000000001516554100600330520ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/x/x/000077500000000000000000000000001516554100600326255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_src/x/x/x/x/x/foo000066400000000000000000000000001516554100600333210ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/000077500000000000000000000000001516554100600320045ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/TARGETS000066400000000000000000000000031516554100600330310ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/a/000077500000000000000000000000001516554100600322245ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/a/b/000077500000000000000000000000001516554100600324455ustar00rootroot00000000000000targets_here/000077500000000000000000000000001516554100600350425ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/a/bTARGETS000066400000000000000000000000031516554100600360670ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/a/b/targets_here{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/bad_targets/000077500000000000000000000000001516554100600342635ustar00rootroot00000000000000TARGETS000066400000000000000000000007071516554100600352440ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/bad_targets{ "string field": { "type": ["simple_rules", "text file"] , "name": "PlAiN sTrInG" , "content": ["This is FOO!"] } , "string field 2": { "type": ["simple_rules", "text file"] , "name": ["OK", 4711, "OK"] , "content": ["This is FOO!"] } , "config field": { "type": ["simple_rules", "transition FOO"] , "value": [{"type": "singleton_map", "key": "FooKey", "value": "BarValue"}] , "deps": [["siple_targets", "rule provides FOO"]] } } config_targets/000077500000000000000000000000001516554100600347235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targetsTARGETS000066400000000000000000000010111516554100600357500ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/config_targets{ "indirect dependency": {"type": ["simple_rules", "upper case"], "srcs": ["use foo"]} , "use foo": { "type": ["simple_rules", "text file"] , "arguments_config": ["foo"] , "name": ["foo.txt."] , "content": [{"type": "var", "name": "foo"}] } , "bar in foo": { "type": "configure" , "arguments_config": ["bar"] , "target": "use foo" , "config": { "type": "let*" , "bindings": [["foo", {"type": "var", "name": "bar", "default": "bar"}]] , "body": {"type": "env", "vars": ["foo"]} } } } file_reference/000077500000000000000000000000001516554100600346625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targetsTARGETS000066400000000000000000000006111516554100600357140ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/file_reference{ "_hello.txt": { "type": "install" , "files": {"raw_data/hello.txt": ["FILE", null, "hello.txt"]} } , "hello.txt": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["_hello.txt"] , "outs": ["hello.txt"] , "cmds": ["cat raw_data/hello.txt | tr 'a-z' 'A-Z' > hello.txt"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/result/000077500000000000000000000000001516554100600333225ustar00rootroot00000000000000TARGETS000066400000000000000000000006121516554100600342760ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/result{ "artifacts": {"type": ["result", "wrong RESULT"], "artifacts": ["YES"]} , "artifacts entry": {"type": ["result", "wrong RESULT"], "artifacts_entry": ["YES"]} , "runfiles": {"type": ["result", "wrong RESULT"], "runfiles": ["YES"]} , "runfiles entry": {"type": ["result", "wrong RESULT"], "runfiles_entry": ["YES"]} , "provides": {"type": ["result", "wrong RESULT"], "provides": ["YES"]} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/simple_rules/000077500000000000000000000000001516554100600345075ustar00rootroot00000000000000TARGETS000066400000000000000000000000031516554100600354550ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/simple_rules{} simple_targets/000077500000000000000000000000001516554100600347475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targetsTARGETS000066400000000000000000000101351516554100600360030ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/simple_targets{ "rule just provides": {"type": ["simple_rules", "just provide"]} , "rule provides FOO": {"type": ["simple_rules", "provide variable FOO"]} , "config transition for FOO": { "type": ["simple_rules", "transition FOO"] , "value": ["bar", "baz"] , "deps": ["rule provides FOO"] } , "collect dep artifacts": { "type": ["simple_rules", "collect deps"] , "deps": ["bar.txt", "baz.txt", "foo.txt", "link"] } , "collect as runfiles": { "type": ["simple_rules", "collect deps as runfiles"] , "deps": ["bar.txt", "baz.txt", "foo.txt", "link"] } , "stage blob": { "type": ["simple_rules", "text file"] , "name": ["foo.txt"] , "content": ["This is FOO!"] } , "stage link": { "type": ["simple_rules", "symlink"] , "name": ["foo.txt"] , "content": ["this/is/a/link"] } , "bad absolute link": { "type": ["simple_rules", "symlink"] , "name": ["foo.txt"] , "content": ["/absolute/link"] } , "bad upwards link": { "type": ["simple_rules", "symlink"] , "name": ["foo.txt"] , "content": ["../upwards/link"] } , "use implicit": {"type": ["simple_rules", "implicit file"]} , "actions": {"type": ["simple_rules", "upper case"], "srcs": ["foo.txt", "bar.txt"]} , "artifact names": { "type": ["simple_rules", "text file"] , "name": ["index.txt"] , "content": [ { "type": "join" , "separator": ";" , "$1": {"type": "outs", "dep": "collect dep artifacts"} } ] , "analyze": ["collect dep artifacts"] } , "runfile names": { "type": ["simple_rules", "text file"] , "name": ["index.txt"] , "content": [ { "type": "join" , "separator": ";" , "$1": {"type": "runfiles", "dep": "collect as runfiles"} } ] , "analyze": ["collect as runfiles"] } , "use generic sym": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["bar.txt"] , "cmds": ["ln -s $(cat bar.txt) sym"] , "outs": ["sym"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "use generic": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["foo.txt", "link"] , "cmds": ["cat foo.txt > out", "readlink link > out", "echo 'DONE' >> out"] , "outs": ["out"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "install": { "type": "install" , "deps": ["bar.txt", "foo.txt", "link"] , "files": { "link_gen": "use generic sym" , "combined.txt": "use generic" , "subdir/restaged.txt": "bar.txt" } , "dirs": [ ["collect as runfiles", "mix/in/this/subdir"] , ["runfile names", "mix/in/this/subdir"] ] } , "generate file": {"type": "file_gen", "name": "generated.txt", "data": "Hello World!"} , "generate symlink": {"type": "symlink", "name": "generated_link", "data": "dummy_link_target"} , "ok outs": { "type": ["simple_rules", "action"] , "cmd": ["sh", "-c", "mkdir foo/bar && echo hello > foo/bar/baz.txt"] , "outs": ["foo/bar/baz.txt"] } , "bad outs": { "type": ["simple_rules", "action"] , "cmd": ["sh", "-c", "echo hello > ../hello.txt"] , "outs": ["../hello.txt"] } , "non-normal outs and out_dirs": { "type": ["simple_rules", "action"] , "cmd": [ "sh" , "-c" , "mkdir -p install && echo Hello > install/data.txt && echo OK > log" ] , "outs": ["./log"] , "out_dirs": ["install/."] } , "generic non-normal outs and out_dirs": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "cmds": ["mkdir -P install", "echo Hello > install/data", "echo OK > log"] , "outs": ["./log"] , "out_dirs": ["install/."] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "stage conflict: rule": { "type": ["simple_rules", "action"] , "cmd": ["sh", "-c", "mkdir -p install && echo Hello > install/data.txt"] , "outs": ["./install/data.txt"] , "out_dirs": ["install/."] } , "stage conflict: generic": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "cmds": ["mkdir -p install && echo Hello > install/data.txt"] , "outs": ["./install/data.txt"] , "out_dirs": ["install/."] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } } symlink_reference/000077500000000000000000000000001516554100600354315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targetsTARGETS000066400000000000000000000005511516554100600364660ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/symlink_reference{ "_link": {"type": "install", "files": {"raw_data/link": ["SYMLINK", null, "link"]}} , "link": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["_link"] , "outs": ["link"] , "cmds": ["ln -s $(readlink raw_data/link | tr 'a-z' 'A-Z') link"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/tree/000077500000000000000000000000001516554100600327435ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/tree/TARGETS000066400000000000000000000003411516554100600337750ustar00rootroot00000000000000{ "no conflict": { "type": ["tree", "ls -R"] , "tree": [["simple_targets", "collect as runfiles"]] , "direct": ["foo.txt"] } , "range conflict": {"type": ["tree", "ls -R"], "tree": [], "direct": ["tree/foo.txt"]} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/tree_overlay/000077500000000000000000000000001516554100600345045ustar00rootroot00000000000000TARGETS000066400000000000000000000007231516554100600354630ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/tree_overlay{ "empty": {"type": ["tree_overlay", "overlay"]} , "one stage": {"type": ["tree_overlay", "overlay"], "deps": ["x"]} , "disjoint empty": {"type": ["tree_overlay", "disjoint overlay"]} , "disjoint one stage": {"type": ["tree_overlay", "disjoint overlay"], "deps": ["x"]} , "built-in, one stage": {"type": "tree_overlay", "name": "the_tree", "deps": ["x"]} , "built-in, disjoint, one stage": {"type": "disjoint_tree_overlay", "name": "the_tree", "deps": ["x"]} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/000077500000000000000000000000001516554100600322535ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/TARGETS000066400000000000000000000000031516554100600333000ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/000077500000000000000000000000001516554100600325225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/TARGETS000066400000000000000000000000031516554100600335470ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/000077500000000000000000000000001516554100600327715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/TARGETS000066400000000000000000000002571516554100600340310ustar00rootroot00000000000000{ "addressing": { "type": "install" , "files": { "absolute": ["x/x", "foo"] , "relative": ["./", "x/x", "foo"] , "upwards": ["./", "../..", "foo"] } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/x/000077500000000000000000000000001516554100600332405ustar00rootroot00000000000000TARGETS000066400000000000000000000000031516554100600342060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/x{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/x/x/000077500000000000000000000000001516554100600335075ustar00rootroot00000000000000TARGETS000066400000000000000000000000031516554100600344550ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/data_targets/x/x/x/x/x{} just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/result_map.test.cpp000066400000000000000000000164121516554100600332030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/result_map.hpp" #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/progress_reporting/progress.hpp" namespace { [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/build_engine/target_map"; } [[nodiscard]] auto CreateAnalysedTarget( TargetResult const& result, std::vector const& descs, std::vector const& blobs) -> AnalysedTargetPtr { return std::make_shared( result, descs, blobs, std::vector(), std::vector(), std::unordered_set{}, std::set{}, std::set{}, TargetGraphInformation::kSource); } } // namespace TEST_CASE("empty map", "[result_map]") { using BuildMaps::Target::ResultTargetMap; ResultTargetMap map{0}; Statistics stats{}; Progress progress{}; CHECK(map.ToResult(&stats, &progress).actions.empty()); CHECK(map.ToResult(&stats, &progress).blobs.empty()); CHECK(map.ToJson(&stats, &progress) == R"({"actions": {}, "blobs": [], "trees": {}})"_json); auto filename = (GetTestDir() / "test_empty.graph").string(); map.ToFile({std::filesystem::path(filename)}, &stats, &progress); std::ifstream file(filename); nlohmann::json from_file{}; file >> from_file; CHECK(from_file == R"({"actions": {}, "blobs": [], "trees": {}})"_json); } TEST_CASE("origins creation", "[result_map]") { using BuildMaps::Base::EntityName; using BuildMaps::Target::ResultTargetMap; auto foo = std::make_shared( ActionDescription::outputs_t{}, ActionDescription::outputs_t{}, Action{"run_foo", {"touch", "foo"}, {}}, ActionDescription::inputs_t{}); auto bar = std::make_shared( ActionDescription::outputs_t{}, ActionDescription::outputs_t{}, Action{"run_bar", {"touch", "bar"}, {}}, ActionDescription::inputs_t{}); auto baz = std::make_shared( ActionDescription::outputs_t{}, ActionDescription::outputs_t{}, Action{"run_baz", {"touch", "baz"}, {}}, ActionDescription::inputs_t{}); ResultTargetMap map{0}; CHECK(map.Add(EntityName{"", ".", "foobar"}, {}, CreateAnalysedTarget( {}, std::vector{foo, bar}, {}))); CHECK(map.Add(EntityName{"", ".", "baz"}, {}, CreateAnalysedTarget( {}, std::vector{baz}, {}))); Statistics stats{}; Progress progress{}; auto result = map.ToResult(&stats, &progress); REQUIRE(result.actions.size() == 3); CHECK(result.blobs.empty()); auto expect_foo = foo->ToJson(); auto expect_bar = bar->ToJson(); auto expect_baz = baz->ToJson(); CHECK(map.ToJson(&stats, &progress) == nlohmann::json{{"actions", {{foo->Id(), expect_foo}, {bar->Id(), expect_bar}, {baz->Id(), expect_baz}}}, {"blobs", nlohmann::json::array()}, {"trees", nlohmann::json::object()}}); expect_foo["origins"] = R"([{"target": ["@", "", "", "foobar"], "config": {}, "subtask": 0}])"_json; expect_bar["origins"] = R"([{"target": ["@", "", "", "foobar"], "config": {}, "subtask": 1}])"_json; expect_baz["origins"] = R"([{"target": ["@", "", "", "baz"], "config": {}, "subtask": 0}])"_json; auto filename = (GetTestDir() / "test_with_origins.graph").string(); map.ToFile({std::filesystem::path(filename)}, &stats, &progress); std::ifstream file(filename); nlohmann::json from_file{}; file >> from_file; CHECK(from_file == nlohmann::json{{"actions", {{foo->Id(), expect_foo}, {bar->Id(), expect_bar}, {baz->Id(), expect_baz}}}, {"blobs", nlohmann::json::array()}, {"trees", nlohmann::json::object()}}); } TEST_CASE("blobs uniqueness", "[result_map]") { using BuildMaps::Base::EntityName; using BuildMaps::Target::ResultTargetMap; ResultTargetMap map{0}; CHECK(map.Add(EntityName{"", ".", "foobar"}, {}, CreateAnalysedTarget({}, {}, {"foo", "bar"}))); CHECK(map.Add(EntityName{"", ".", "barbaz"}, {}, CreateAnalysedTarget({}, {}, {"bar", "baz"}))); Statistics stats{}; Progress progress{}; auto result = map.ToResult(&stats, &progress); CHECK(result.actions.empty()); CHECK(result.blobs.size() == 3); CHECK(map.ToJson(&stats, &progress) == nlohmann::json{{"actions", nlohmann::json::object()}, {"blobs", {"bar", "baz", "foo"}}, {"trees", nlohmann::json::object()}}); auto filename = (GetTestDir() / "test_unique_blobs.graph").string(); map.ToFile( {std::filesystem::path(filename)}, &stats, &progress); std::ifstream file(filename); nlohmann::json from_file{}; file >> from_file; CHECK(from_file == nlohmann::json{{"actions", nlohmann::json::object()}, {"blobs", {"bar", "baz", "foo"}}, {"trees", nlohmann::json::object()}}); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map/target_map.test.cpp000066400000000000000000002443631516554100600331630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/target_map.hpp" #include #include #include #include #include #include // std::move #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" #include "src/buildtool/build_engine/base_maps/directory_map.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/base_maps/expression_map.hpp" #include "src/buildtool/build_engine/base_maps/rule_map.hpp" #include "src/buildtool/build_engine/base_maps/source_map.hpp" #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/result_map.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/main/analyse_context.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/serve_service/test_serve_config.hpp" namespace { using none_t = Expression::none_t; auto CreateSymlinks() -> bool { auto base_src = std::filesystem::path( "test/buildtool/build_engine/target_map/data_src"); // create the symlinks REQUIRE(FileSystemManager::CreateSymlink( "dummy", base_src / "a" / "b" / "targets_here" / "c" / "d" / "link")); REQUIRE(FileSystemManager::CreateSymlink( "dummy", base_src / "symlink_reference" / "link")); REQUIRE(FileSystemManager::CreateSymlink( "dummy", base_src / "simple_targets" / "link")); return true; } auto SetupConfig() -> RepositoryConfig { // manually create locally test symlinks in data_src, but only once [[maybe_unused]] static auto done = CreateSymlinks(); // create the file roots auto info = RepositoryConfig::RepositoryInfo{ FileRoot{std::filesystem::path{"test/buildtool/build_engine/target_map/" "data_src"}}, FileRoot{std::filesystem::path{"test/buildtool/build_engine/target_map/" "data_targets"}}, FileRoot{std::filesystem::path{"test/buildtool/build_engine/target_map/" "data_rules"}}, FileRoot{std::filesystem::path{"test/buildtool/build_engine/target_map/" "data_expr"}}}; RepositoryConfig repo_config{}; repo_config.SetInfo("", std::move(info)); return repo_config; } } // namespace TEST_CASE("simple targets", "[target_map]") { // NOLINT auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("Actual source file") { { error_msg = "NONE"; error = false; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "a/b/targets_here", "c/d/foo"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); auto artifacts = result->Artifacts(); ExpressionPtr artifact = artifacts->Get("c/d/foo", none_t{}); CHECK(artifact->IsArtifact()); } SECTION("Actual source symlink file") { { error_msg = "NONE"; error = false; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "a/b/targets_here", "c/d/link"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); auto artifacts = result->Artifacts(); ExpressionPtr artifact = artifacts->Get("c/d/link", none_t{}); CHECK(artifact->IsArtifact()); } SECTION("No targets file here") { { error_msg = "NONE"; error = false; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "a/b/targets_here/c", "d/foo"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); } SECTION("Rule just provides") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "rule just provides"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Provides() == Expression::FromJson(R"({"foo": "bar"})"_json)); } SECTION("Rule provides variable, but unset") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "rule provides FOO"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Provides() == Expression::FromJson(R"({"foo": null})"_json)); } SECTION("Rule provides variable, set in config") { { error_msg = "NONE"; error = false; result = nullptr; auto config = Configuration{ Expression::FromJson(R"({"FOO": "foobar"})"_json)}; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "rule provides FOO"}, .config = config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Provides() == Expression::FromJson(R"({"foo": "foobar"})"_json)); } SECTION("Rule provides variable, set via config transition") { { error_msg = "NONE"; error = false; result = nullptr; auto config = Configuration{ Expression::FromJson(R"({"FOO": "foobar"})"_json)}; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "config transition for FOO"}, .config = config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK( result->Provides() == Expression::FromJson(R"({"transitioned deps": ["barbaz"]})"_json)); } SECTION("Rule collects dependency artifacts") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "collect dep artifacts"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); // Look into the internals of the artifacts by using the json // representation auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["foo.txt"]["data"]["path"] == "simple_targets/foo.txt"); CHECK(artifacts_desc["bar.txt"]["data"]["path"] == "simple_targets/bar.txt"); CHECK(artifacts_desc["baz.txt"]["data"]["path"] == "simple_targets/baz.txt"); CHECK(artifacts_desc["link"]["data"]["path"] == "simple_targets/link"); } SECTION("Rule stages blob") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "stage blob"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); auto blobs = result->Blobs(); CHECK(blobs.size() == 1); CHECK(blobs[0] == "This is FOO!"); auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["foo.txt"]["type"] == "KNOWN"); } SECTION("Rule stages symlink") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "stage link"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); auto blobs = result->Blobs(); CHECK(blobs.size() == 1); CHECK(blobs[0] == "this/is/a/link"); auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["foo.txt"]["type"] == "KNOWN"); CHECK(artifacts_desc["foo.txt"]["data"]["file_type"] == "l"); } SECTION("Rule stages symlink, bad: absolute") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "bad absolute link"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); } SECTION("Rule stages symlink, bad: upwards") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "bad upwards link"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); } SECTION("Stage implicit target") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "use implicit"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); // Look into the internals of the artifacts by using the json // representation auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["implicit_script.sh"]["data"]["path"] == "simple_rules/implicit_script.sh"); } SECTION("simple actions") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "actions"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); // Look into the internals of the artifacts by using the json // representation auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["foo.txt"]["type"] == "ACTION"); CHECK(artifacts_desc["bar.txt"]["type"] == "ACTION"); // We have a deterministic evaluation order, so the order of the actions // in the vector is guaranteed. The test rule generates the action by // iterating over the "srcs" field, so we get the actions in the order // of that field, not in alphabetical order. CHECK(result->Actions()[0]->ToJson()["input"]["in"]["data"]["path"] == "simple_targets/foo.txt"); CHECK(result->Actions()[1]->ToJson()["input"]["in"]["data"]["path"] == "simple_targets/bar.txt"); } SECTION("ok outs") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "ok outs"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); auto artifacts_desc = result->Artifacts()->ToJson(); CHECK(artifacts_desc["foo/bar/baz.txt"]["type"] == "ACTION"); } SECTION("bad outs") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "bad outs"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); } SECTION("non-normal outs and out_dirs") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "simple_targets", "non-normal outs and " "out_dirs"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); REQUIRE(result->Actions().size() == 1); auto action = result->Actions()[0]->ToJson(); CHECK(action["output_dirs"] == R"(["install"])"_json); CHECK(action["output"] == R"(["log"])"_json); } SECTION("generic non-normal outs and out_dirs") { { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "simple_targets", "generic non-normal " "outs and out_dirs"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); REQUIRE(result->Actions().size() == 1); auto action = result->Actions()[0]->ToJson(); CHECK(action["output_dirs"] == R"(["install"])"_json); CHECK(action["output"] == R"(["log"])"_json); } } SECTION("staging conflict: rule") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "stage conflict: rule"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("artifacts") != std::string::npos); CHECK(error_msg.find("install") != std::string::npos); } SECTION("staging conflict: generic") { { error_msg = "NONE"; error = false; result = nullptr; TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "stage conflict: generic"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("artifacts") != std::string::npos); CHECK(error_msg.find("install") != std::string::npos); } } TEST_CASE("configuration deduplication", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); std::vector result; bool error{false}; std::string error_msg = "NONE"; auto config = Configuration{Expression::FromJson( R"({"foo" : "bar", "irrelevant": "ignore me"})"_json)}; auto alternative_config = Configuration{Expression::FromJson( R"({"foo" : "bar", "irrelevant": "other value"})"_json)}; auto different_config = Configuration{Expression::FromJson(R"({"foo" : "baz"})"_json)}; auto indirect_target = BuildMaps::Base::EntityName{ "", "config_targets", "indirect dependency"}; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{.target = indirect_target, .config = config}, BuildMaps::Target::ConfiguredTarget{.target = indirect_target, .config = alternative_config}, BuildMaps::Target::ConfiguredTarget{.target = indirect_target, .config = different_config}}, [&result](auto values) { std::transform(values.begin(), values.end(), std::back_inserter(result), [](auto* target) { return *target; }); }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result[0]->Artifacts() == result[1]->Artifacts()); CHECK(result[0]->Artifacts() != result[2]->Artifacts()); Progress progress{}; auto analysis_result = result_map.ToResult(&stats, &progress); CHECK(analysis_result.actions.size() == 2); } TEST_CASE("generator functions in string arguments", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("outs") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "artifact names"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["index.txt"]["type"] == "KNOWN"); CHECK(result->Blobs()[0] == "bar.txt;baz.txt;foo.txt;link"); } SECTION("runfies") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "runfile names"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["index.txt"]["type"] == "KNOWN"); CHECK(result->Blobs()[0] == "bar.txt;baz.txt;foo.txt;link"); } } TEST_CASE("built-in rules", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("generic") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "use generic"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->Map().size() == 1); CHECK(result->Artifacts()->ToJson()["out"]["type"] == "ACTION"); CHECK(result->Artifacts()->ToJson()["out"]["data"]["path"] == "out"); } SECTION("install") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "install"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts() == result->RunFiles()); auto stage = result->Artifacts()->ToJson(); CHECK(stage["foo.txt"]["type"] == "LOCAL"); CHECK(stage["foo.txt"]["data"]["path"] == "simple_targets/foo.txt"); CHECK(stage["bar.txt"]["type"] == "LOCAL"); CHECK(stage["bar.txt"]["data"]["path"] == "simple_targets/bar.txt"); CHECK(stage["bar.txt"]["type"] == "LOCAL"); CHECK(stage["link"]["data"]["path"] == "simple_targets/link"); CHECK(stage["combined.txt"]["type"] == "ACTION"); CHECK(stage["combined.txt"]["data"]["path"] == "out"); CHECK(stage["link_gen"]["type"] == "ACTION"); CHECK(stage["link_gen"]["data"]["path"] == "sym"); CHECK(stage["subdir/restaged.txt"]["type"] == "LOCAL"); CHECK(stage["subdir/restaged.txt"]["data"]["path"] == "simple_targets/bar.txt"); CHECK(stage["mix/in/this/subdir/foo.txt"]["data"]["path"] == "simple_targets/foo.txt"); CHECK(stage["mix/in/this/subdir/bar.txt"]["data"]["path"] == "simple_targets/bar.txt"); CHECK(stage["mix/in/this/subdir/baz.txt"]["data"]["path"] == "simple_targets/baz.txt"); CHECK(stage["mix/in/this/subdir/link"]["data"]["path"] == "simple_targets/link"); CHECK(stage["mix/in/this/subdir/index.txt"]["type"] == "KNOWN"); } SECTION("file_gen") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "generate file"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["generated.txt"]["type"] == "KNOWN"); CHECK(result->Blobs().size() == 1); CHECK(result->Blobs()[0] == "Hello World!"); } SECTION("symlink") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "simple_targets", "generate symlink"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["generated_link"]["type"] == "KNOWN"); CHECK(result->Blobs().size() == 1); CHECK(result->Blobs()[0] == "dummy_link_target"); } SECTION("configure") { auto target = BuildMaps::Base::EntityName{"", "config_targets", "bar in foo"}; auto baz_config = empty_config.Update("bar", ExpressionPtr{std::string{"baz"}}); error = false; error_msg = "NONE"; AnalysedTargetPtr bar_result{}; AnalysedTargetPtr baz_result{}; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{.target = target, .config = empty_config}, BuildMaps::Target::ConfiguredTarget{.target = target, .config = baz_config}}, [&bar_result, &baz_result](auto values) { bar_result = *values[0]; baz_result = *values[1]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(bar_result->Artifacts()->ToJson()["foo.txt."]["type"] == "KNOWN"); CHECK( bar_result->Artifacts()->ToJson()["foo.txt."]["data"]["id"] == storage_config.Get().hash_function.HashBlobData("bar").HexString()); CHECK(baz_result->Artifacts()->ToJson()["foo.txt."]["type"] == "KNOWN"); CHECK( baz_result->Artifacts()->ToJson()["foo.txt."]["data"]["id"] == storage_config.Get().hash_function.HashBlobData("baz").HexString()); } SECTION("tree_overlay") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree_overlay", "built-in, one stage"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["the_tree"]["type"] == "TREE_OVERLAY"); CHECK(result->Trees().size() == 1); CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == "TREE"); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == false); } SECTION("disjoint_tree_overlay") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "tree_overlay", "built-in, disjoint, " "one stage"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["the_tree"]["type"] == "TREE_OVERLAY"); CHECK(result->Trees().size() == 1); CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == "TREE"); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == true); } } TEST_CASE("target reference", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("file vs target") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "file_reference", "hello.txt"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["hello.txt"]["type"] == "ACTION"); CHECK(result->Artifacts()->ToJson()["hello.txt"]["data"]["path"] == "hello.txt"); CHECK(result->Actions().size() == 1); CHECK(result->Actions()[0] ->ToJson()["input"]["raw_data/hello.txt"]["type"] == "LOCAL"); CHECK(result->Actions()[0] ->ToJson()["input"]["raw_data/hello.txt"]["data"]["path"] == "file_reference/hello.txt"); } SECTION("symlink vs target") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "symlink_reference", "link"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["link"]["type"] == "ACTION"); CHECK(result->Artifacts()->ToJson()["link"]["data"]["path"] == "link"); CHECK(result->Actions().size() == 1); CHECK( result->Actions()[0]->ToJson()["input"]["raw_data/link"]["type"] == "LOCAL"); CHECK(result->Actions()[0] ->ToJson()["input"]["raw_data/link"]["data"]["path"] == "symlink_reference/link"); } SECTION("relative address") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "x/x/x", "addressing"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Artifacts()->ToJson()["absolute"]["data"]["path"] == "x/x/foo"); CHECK(result->Artifacts()->ToJson()["relative"]["data"]["path"] == "x/x/x/x/x/foo"); CHECK(result->Artifacts()->ToJson()["upwards"]["data"]["path"] == "x/foo"); } } TEST_CASE("trees", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("no conflict") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "tree", "no conflict"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Actions().size() == 1); CHECK(result->Actions()[0]->ToJson()["input"]["tree"]["type"] == "TREE"); CHECK(result->Actions()[0]->ToJson()["input"]["foo.txt"]["type"] == "LOCAL"); CHECK(result->Actions()[0]->ToJson()["input"]["foo.txt"]["data"]["pat" "h"] == "tree/foo.txt"); CHECK(result->Trees().size() == 1); CHECK(result->Trees()[0]->ToJson()["foo.txt"]["type"] == "LOCAL"); CHECK(result->Trees()[0]->ToJson()["bar.txt"]["type"] == "LOCAL"); CHECK(result->Trees()[0]->ToJson()["baz.txt"]["type"] == "LOCAL"); } SECTION("stage into tree") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree", "range conflict"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); } } TEST_CASE("tree_overlays", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("empty") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree_overlay", "empty"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"] == nlohmann::json::array()); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == false); } SECTION("implicit tree") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree_overlay", "one stage"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Trees().size() == 1); CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == "TREE"); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == false); } SECTION("disjoint empty") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree_overlay", "disjoint empty"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"] == nlohmann::json::array()); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == true); } SECTION("disjoint implicit tree") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "tree_overlay", "disjoint one stage"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(not error); CHECK(error_msg == "NONE"); CHECK(result->Trees().size() == 1); CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); CHECK(result->TreeOverlays().size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == "TREE"); CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == true); } } TEST_CASE("RESULT error reporting", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("artifacts") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "result", "artifacts"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("artifacts-not-a-map") != std::string::npos); } SECTION("artifacts entry") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "result", "artifacts entry"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("bad-artifact-entry") != std::string::npos); CHECK(error_msg.find("bad-artifact-path") != std::string::npos); } SECTION("runfiles") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "result", "runfiles"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("runfiles-not-a-map") != std::string::npos); } SECTION("runfiles entry") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "result", "runfiles entry"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("bad-runfiles-entry") != std::string::npos); CHECK(error_msg.find("bad-runfiles-path") != std::string::npos); } SECTION("provides") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{"", "result", "provides"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("provides-not-a-map") != std::string::npos); } } TEST_CASE("wrong arguments", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto repo_config = SetupConfig(); auto directory_entries = BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); auto source = BuildMaps::Base::CreateSourceTargetMap( &directory_entries, &repo_config, storage_config.Get().hash_function.GetType()); auto targets_file_map = BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); static auto expressions_file_map = BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, &repo_config); auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); BuildMaps::Target::ResultTargetMap result_map{0}; Statistics stats{}; Progress exports_progress{}; auto serve_config = TestServeConfig::ReadFromEnvironment(); REQUIRE(serve_config); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig remote_exec_config{}; // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_exec_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, /*repo_config=*/nullptr); auto serve = ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); AnalyseContext ctx{.repo_config = &repo_config, .storage = &storage, .statistics = &stats, .progress = &exports_progress, .serve = serve ? &*serve : nullptr}; auto absent_target_variables_map = BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( &ctx, &result_map, &absent_target_variables_map, 0); auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, &source, &targets_file_map, &rule_map, &directory_entries, &absent_target_map, &result_map); AnalysedTargetPtr result; bool error{false}; std::string error_msg; auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; SECTION("string field") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "bad_targets", "string field"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("PlAiN sTrInG") != std::string::npos); } SECTION("string field 2") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "bad_targets", "string field 2"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("4711") != std::string::npos); } SECTION("config field") { error = false; error_msg = "NONE"; { TaskSystem ts; target_map.ConsumeAfterKeysReady( &ts, {BuildMaps::Target::ConfiguredTarget{ .target = BuildMaps::Base::EntityName{ "", "bad_targets", "config field"}, .config = empty_config}}, [&result](auto values) { result = *values[0]; }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK(error); CHECK(error_msg != "NONE"); CHECK(error_msg.find("FooKey") != std::string::npos); CHECK(error_msg.find("BarValue") != std::string::npos); } } target_map_internals.test.cpp000066400000000000000000000054451516554100600351570ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/build_engine/target_map// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/build_engine/target_map/utils.hpp" TEST_CASE("Tree conflicts", "[tree_conflict]") { auto overlap = Expression::FromJson(R"({ "foo/bar": "content-1" , "foo/bar/baz": "content-2"})"_json); REQUIRE(overlap); auto overlap_conflict = BuildMaps::Target::Utils::tree_conflict(overlap); CHECK(overlap_conflict); CHECK(*overlap_conflict == "foo/bar"); auto root_overlap = Expression::FromJson(R"({ ".": "content-1" , "foo": "content-2"})"_json); REQUIRE(root_overlap); auto root_overlap_conflict = BuildMaps::Target::Utils::tree_conflict(root_overlap); CHECK(root_overlap_conflict); CHECK(*root_overlap_conflict == "."); auto upwards_reference = Expression::FromJson(R"( { "../foo.txt" : "content" })"_json); REQUIRE(upwards_reference); auto upwards_reference_conflict = BuildMaps::Target::Utils::tree_conflict(upwards_reference); CHECK(upwards_reference_conflict); CHECK(*upwards_reference_conflict == "../foo.txt"); auto absolute_reference = Expression::FromJson(R"( { "/foo.txt" : "content" })"_json); REQUIRE(absolute_reference); auto absolute_reference_conflict = BuildMaps::Target::Utils::tree_conflict(absolute_reference); CHECK(absolute_reference_conflict); CHECK(*absolute_reference_conflict == "/foo.txt"); } TEST_CASE("No conflict", "[tree_conflict]") { auto no_overlap = Expression::FromJson(R"({ "foo/bar/baz.txt": "content-1" , "foo/bar/baz": "content-2"})"_json); REQUIRE(no_overlap); auto no_overlap_conflict = BuildMaps::Target::Utils::tree_conflict(no_overlap); CHECK(not no_overlap_conflict); auto single_root = Expression::FromJson(R"({ ".": "content-1"})"_json); REQUIRE(single_root); auto single_root_conflict = BuildMaps::Target::Utils::tree_conflict(single_root); CHECK(not single_root_conflict); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/000077500000000000000000000000001516554100600240635ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/TARGETS000066400000000000000000000041341516554100600251210ustar00rootroot00000000000000{ "artifact_description": { "type": ["@", "rules", "CC/test", "test"] , "name": ["artifact_description"] , "srcs": ["artifact_description.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "common"] } , "action_description": { "type": ["@", "rules", "CC/test", "test"] , "name": ["action_description"] , "srcs": ["action_description.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "common"] } , "repository_config": { "type": ["@", "rules", "CC/test", "test"] , "name": ["repository_config"] , "srcs": ["repository_config.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "common"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["common"] , "deps": [ "action_description" , "artifact_description" , "repository_config" , ["./", "remote", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/action_description.test.cpp000066400000000000000000000072711516554100600314340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/action_description.hpp" #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" TEST_CASE("From JSON", "[action_description]") { using path = std::filesystem::path; auto desc = ActionDescription{ {"output0", "output1"}, {"dir0", "dir1"}, Action{"id", {"command", "line"}, {{"env", "vars"}}}, {{"path0", ArtifactDescription::CreateTree(path{"input0"})}, {"path1", ArtifactDescription::CreateTree(path{"input1"})}}}; auto const& action = desc.GraphAction(); auto json = ActionDescription{desc.OutputFiles(), desc.OutputDirs(), Action{"unused", action.Command(), action.Env()}, desc.Inputs()} .ToJson(); auto const hash_type = TestHashType::ReadFromEnvironment(); SECTION("Parse full action") { auto description = ActionDescription::FromJson(hash_type, "id", json); REQUIRE(description); CHECK((*description)->ToJson() == json); } SECTION("Parse action without optional input") { json["input"] = nlohmann::json::object(); CHECK(ActionDescription::FromJson(hash_type, "id", json)); json["input"] = nlohmann::json::array(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("input"); CHECK(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without optional env") { json["env"] = nlohmann::json::object(); CHECK(ActionDescription::FromJson(hash_type, "id", json)); json["env"] = nlohmann::json::array(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("env"); CHECK(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without mandatory outputs") { json["output"] = nlohmann::json::array(); json["output_dirs"] = nlohmann::json::array(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json["output"] = nlohmann::json::object(); json["output_dirs"] = nlohmann::json::object(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("output"); json.erase("output_dirs"); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without mandatory command") { json["command"] = nlohmann::json::array(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json["command"] = nlohmann::json::object(); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("command"); CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/artifact_description.test.cpp000066400000000000000000000220041516554100600317430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/artifact_description.hpp" #include #include #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" static auto NamedDigest(std::string const& str) -> ArtifactDigest { HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; return ArtifactDigestFactory::HashDataAs(hash_function, str); } [[nodiscard]] auto operator==(Artifact const& lhs, Artifact const& rhs) -> bool { return lhs.Id() == rhs.Id() and lhs.FilePath() == rhs.FilePath() and lhs.Info() == rhs.Info(); } TEST_CASE("Local artifact", "[artifact_description]") { auto local_desc = ArtifactDescription::CreateLocal( std::filesystem::path{"local_path"}, "repo"); auto from_json = ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), local_desc.ToJson()); CHECK(local_desc == *from_json); } TEST_CASE("Known artifact", "[artifact_description]") { SECTION("File object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("f_fake_hash"), ObjectType::File); auto from_json = ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_desc.ToJson()); CHECK(known_desc == *from_json); } SECTION("Executable object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("x_fake_hash"), ObjectType::Executable); auto from_json = ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_desc.ToJson()); CHECK(known_desc == *from_json); } SECTION("Symlink object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("l_fake_hash"), ObjectType::Symlink); auto from_json = ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_desc.ToJson()); CHECK(known_desc == *from_json); } } TEST_CASE("Action artifact", "[artifact_description]") { auto action_desc = ArtifactDescription::CreateAction( "action_id", std::filesystem::path{"out_path"}); auto from_json = ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), action_desc.ToJson()); CHECK(action_desc == *from_json); } TEST_CASE("From JSON", "[artifact_description]") { auto local = ArtifactDescription::CreateLocal("local", "repo").ToJson(); auto known = ArtifactDescription::CreateKnown(NamedDigest("hash"), ObjectType::File) .ToJson(); auto action = ArtifactDescription::CreateAction("id", "output").ToJson(); auto const hash_type = TestHashType::ReadFromEnvironment(); SECTION("Parse artifacts") { CHECK(ArtifactDescription::FromJson(hash_type, local)); CHECK(ArtifactDescription::FromJson(hash_type, known)); CHECK(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse artifact without mandatory type") { local.erase("type"); known.erase("type"); action.erase("type"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse artifact without mandatory data") { local.erase("data"); known.erase("data"); action.erase("data"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse local artifact without mandatory path") { local["data"]["path"] = 0; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); local["data"].erase("path"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); } SECTION("Parse known artifact") { SECTION("without mandatory id") { known["data"]["id"] = 0; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("id"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } SECTION("without mandatory size") { known["data"]["size"] = "0"; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("size"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } SECTION("without mandatory file_type") { known["data"]["file_type"] = "more_than_one_char"; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("file_type"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } } SECTION("Parse action artifact") { SECTION("without mandatory id") { action["data"]["id"] = 0; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); action["data"].erase("id"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("without mandatory path") { action["data"]["path"] = 0; CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); action["data"].erase("path"); CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } } } TEST_CASE("Description missing mandatory key/value pair", "[artifact_description]") { nlohmann::json const missing_type = {{"data", {{"path", "some/path"}}}}; CHECK(not ArtifactDescription::FromJson(TestHashType::ReadFromEnvironment(), missing_type)); nlohmann::json const missing_data = {{"type", "LOCAL"}}; CHECK(not ArtifactDescription::FromJson(TestHashType::ReadFromEnvironment(), missing_data)); } TEST_CASE("Local artifact description contains incorrect value for \"data\"", "[artifact_description]") { nlohmann::json const local_art_missing_path = { {"type", "LOCAL"}, {"data", nlohmann::json::object()}}; CHECK(not ArtifactDescription::FromJson(TestHashType::ReadFromEnvironment(), local_art_missing_path)); } TEST_CASE("Known artifact description contains incorrect value for \"data\"", "[artifact_description]") { std::string file_type{}; file_type += ToChar(ObjectType::File); SECTION("missing \"id\"") { nlohmann::json const known_art_missing_id = { {"type", "KNOWN"}, {"data", {{"size", 15}, {"file_type", file_type}}}}; CHECK(not ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_art_missing_id)); } SECTION("missing \"size\"") { nlohmann::json const known_art_missing_size = { {"type", "KNOWN"}, {"data", {{"id", "known_input"}, {"file_type", file_type}}}}; CHECK(not ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_art_missing_size)); } SECTION("missing \"file_type\"") { nlohmann::json const known_art_missing_file_type = { {"type", "KNOWN"}, {"data", {{"id", "known_input"}, {"size", 15}}}}; CHECK(not ArtifactDescription::FromJson( TestHashType::ReadFromEnvironment(), known_art_missing_file_type)); } } TEST_CASE("Action artifact description contains incorrect value for \"data\"", "[artifact_description]") { nlohmann::json const action_art_missing_id = { {"type", "ACTION"}, {"data", {{"path", "output/path"}}}}; CHECK(not ArtifactDescription::FromJson(TestHashType::ReadFromEnvironment(), action_art_missing_id)); nlohmann::json const action_art_missing_path = { {"type", "ACTION"}, {"data", {{"id", "action_id"}}}}; CHECK(not ArtifactDescription::FromJson(TestHashType::ReadFromEnvironment(), action_art_missing_path)); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/remote/000077500000000000000000000000001516554100600253565ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/remote/TARGETS000066400000000000000000000007161516554100600264160ustar00rootroot00000000000000{ "remote_common": { "type": ["@", "rules", "CC/test", "test"] , "name": ["remote_common"] , "srcs": ["remote_common.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "common", "remote"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["remote"] , "deps": ["remote_common"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/remote/remote_common.test.cpp000066400000000000000000000036721516554100600317130ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/remote/remote_common.hpp" #include #include #include #include "catch2/catch_test_macros.hpp" TEST_CASE("Test parsing execution properties", "[remote_config]") { SECTION("execution property has colon in value") { std::optional> parsed_result = ParseProperty("image:test.registry:443/test"); CHECK(parsed_result != std::nullopt); CHECK(parsed_result.value().first == "image"); CHECK(parsed_result.value().second == "test.registry:443/test"); } SECTION("execution property has empty value") { std::optional> parsed_result = ParseProperty("image:"); CHECK(parsed_result == std::nullopt); } SECTION("execution property has no colon") { std::optional> parsed_result = ParseProperty("image"); CHECK(parsed_result == std::nullopt); } SECTION("execution property has key:val format") { std::optional> parsed_result = ParseProperty("image:test"); CHECK(parsed_result != std::nullopt); CHECK(parsed_result.value().first == "image"); CHECK(parsed_result.value().second == "test"); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/common/repository_config.test.cpp000066400000000000000000000251011516554100600313100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/common/repository_config.hpp" #include #include #include #include #include #include #include #include #include // std::move #include #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/common"; } [[nodiscard]] auto GetGitRoot(StorageConfig const* storage_config) -> FileRoot { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); REQUIRE(FileSystemManager::CreateDirectory(repo_path)); auto anchor = FileSystemManager::ChangeDirectory(repo_path); if (std::system("git init") == 0 and std::system("git -c user.name='nobody' -c user.email='' " "commit --allow-empty -m'init'") == 0) { auto constexpr kEmptyTreeId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; if (auto root = FileRoot::FromGit(storage_config, repo_path, kEmptyTreeId)) { return std::move(*root); } } return FileRoot{std::filesystem::path{"missing"}}; } [[nodiscard]] auto CreateFixedRepoInfo( StorageConfig const* storage_config, std::map const& bindings = {}, std::string const& tfn = "TARGETS", std::string const& rfn = "RULES", std::string const& efn = "EXPRESSIONS") { static auto const kGitRoot = GetGitRoot(storage_config); return RepositoryConfig::RepositoryInfo{ kGitRoot, kGitRoot, kGitRoot, kGitRoot, bindings, tfn, rfn, efn}; } [[nodiscard]] auto CreateFileRepoInfo( std::map const& bindings = {}, std::string const& tfn = "TARGETS", std::string const& rfn = "RULES", std::string const& efn = "EXPRESSIONS") { static auto const kFileRoot = FileRoot{std::filesystem::path{"file path"}}; return RepositoryConfig::RepositoryInfo{ kFileRoot, kFileRoot, kFileRoot, kFileRoot, bindings, tfn, rfn, efn}; } template [[nodiscard]] auto Copy(T const& x) -> T { return x; } // Read graph from CAS [[nodiscard]] auto ReadGraph(Storage const& storage, ArtifactDigest const& repo_key) -> nlohmann::json { auto const& cas = storage.CAS(); auto blob = cas.BlobPath(repo_key, /*is_executable=*/false); REQUIRE(blob); auto content = FileSystemManager::ReadFile(*blob); REQUIRE(content); return nlohmann::json::parse(*content); } // From [info0, info1, ...] and [bindings0, bindings1, ...] // build graph: {"0": (info0 + bindings0), "1": (info1 + bindings1), ...} [[nodiscard]] auto BuildGraph( std::vector const& infos, std::vector> const& bindings) -> nlohmann::json { auto graph = nlohmann::json::object(); for (std::size_t i{}; i < infos.size(); ++i) { auto entry = infos[i]->BaseContentDescription(); REQUIRE(entry); (*entry)["bindings"] = bindings[i]; graph[std::to_string(i)] = std::move(*entry); } return graph; } } // namespace TEST_CASE("Test missing repository", "[repository_config]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); RepositoryConfig config{}; CHECK(config.Info("missing") == nullptr); CHECK_FALSE(config.RepositoryKey(storage, "missing")); } TEST_CASE("Compute key of fixed repository", "[repository_config]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); RepositoryConfig config{}; SECTION("for single fixed repository") { config.SetInfo("foo", CreateFixedRepoInfo(&storage_config.Get())); auto key = config.RepositoryKey(storage, "foo"); REQUIRE(key); // verify created graph from CAS CHECK(ReadGraph(storage, *key) == BuildGraph({config.Info("foo")}, {{}})); } SECTION("for fixed repositories with same missing dependency") { config.SetInfo( "foo", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz"}})); config.SetInfo( "bar", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz"}})); CHECK_FALSE(config.RepositoryKey(storage, "foo")); CHECK_FALSE(config.RepositoryKey(storage, "bar")); } SECTION("for fixed repositories with different missing dependency") { config.SetInfo( "foo", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz0"}})); config.SetInfo( "bar", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz1"}})); CHECK_FALSE(config.RepositoryKey(storage, "foo")); CHECK_FALSE(config.RepositoryKey(storage, "bar")); } } TEST_CASE("Compute key of file repository", "[repository_config]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); RepositoryConfig config{}; SECTION("for single file repository") { config.SetInfo("foo", CreateFileRepoInfo()); CHECK_FALSE(config.RepositoryKey(storage, "foo")); } SECTION("for graph with leaf dependency as file") { config.SetInfo( "foo", CreateFixedRepoInfo(&storage_config.Get(), {{"bar", "bar"}})); config.SetInfo( "bar", CreateFixedRepoInfo(&storage_config.Get(), {{"baz", "baz"}})); config.SetInfo("baz", CreateFileRepoInfo()); CHECK_FALSE(config.RepositoryKey(storage, "foo")); } } TEST_CASE("Compare key of two repos with same content", "[repository_config]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); RepositoryConfig config{}; // create two different repo infos with same content (baz should be same) config.SetInfo( "foo", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz0"}})); config.SetInfo( "bar", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz1"}})); SECTION("with leaf dependency") { // create duplicate leaf repo info with global name 'baz0' and 'baz1' auto baz = CreateFixedRepoInfo(&storage_config.Get()); config.SetInfo("baz0", Copy(baz)); config.SetInfo("baz1", Copy(baz)); // check if computed key is same auto foo_key = config.RepositoryKey(storage, "foo"); auto bar_key = config.RepositoryKey(storage, "bar"); REQUIRE(foo_key); REQUIRE(bar_key); CHECK(*foo_key == *bar_key); // verify created graph from CAS CHECK(ReadGraph(storage, *foo_key) == BuildGraph({config.Info("foo"), config.Info("baz0")}, {{{"dep", "1"}}, {}})); } SECTION("with cyclic dependency") { // create duplicate cyclic repo info with global name 'baz0' and 'baz1' auto baz = CreateFixedRepoInfo(&storage_config.Get(), {{"foo", "foo"}, {"bar", "bar"}}); config.SetInfo("baz0", Copy(baz)); config.SetInfo("baz1", Copy(baz)); // check if computed key is same auto foo_key = config.RepositoryKey(storage, "foo"); auto bar_key = config.RepositoryKey(storage, "bar"); REQUIRE(foo_key); REQUIRE(bar_key); CHECK(*foo_key == *bar_key); // verify created graph from CAS CHECK(ReadGraph(storage, *foo_key) == BuildGraph({config.Info("foo"), config.Info("baz0")}, {{{"dep", "1"}}, {{"foo", "0"}, {"bar", "0"}}})); } SECTION("with two separate cyclic graphs") { // create two cyclic repo infos producing two separate graphs config.SetInfo( "baz0", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "foo"}})); config.SetInfo( "baz1", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "bar"}})); // check if computed key is same auto foo_key = config.RepositoryKey(storage, "foo"); auto bar_key = config.RepositoryKey(storage, "bar"); REQUIRE(foo_key); REQUIRE(bar_key); CHECK(*foo_key == *bar_key); // verify created graph from CAS CHECK(ReadGraph(storage, *foo_key) == BuildGraph({config.Info("foo")}, {{{"dep", "0"}}})); } SECTION("for graph with leaf repos refering to themselfs") { config.SetInfo( "baz0", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz0"}})); config.SetInfo( "baz1", CreateFixedRepoInfo(&storage_config.Get(), {{"dep", "baz1"}})); // check if computed key is same auto foo_key = config.RepositoryKey(storage, "foo"); auto bar_key = config.RepositoryKey(storage, "bar"); auto baz0_key = config.RepositoryKey(storage, "baz0"); auto baz1_key = config.RepositoryKey(storage, "baz1"); CHECK(foo_key); CHECK(bar_key); CHECK(baz0_key); CHECK(baz1_key); CHECK(*foo_key == *bar_key); CHECK(*bar_key == *baz0_key); CHECK(*baz0_key == *baz1_key); // verify created graph from CAS CHECK(ReadGraph(storage, *foo_key) == BuildGraph({config.Info("foo")}, {{{"dep", "0"}}})); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/crypto/000077500000000000000000000000001516554100600241135ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/crypto/TARGETS000066400000000000000000000022551516554100600251530ustar00rootroot00000000000000{ "hasher": { "type": ["@", "rules", "CC/test", "test"] , "name": ["hasher"] , "srcs": ["hasher.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/crypto", "hasher"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "crypto"] } , "hash_function": { "type": ["@", "rules", "CC/test", "test"] , "name": ["hash_function"] , "srcs": ["hash_function.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/crypto", "hasher"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "crypto"] } , "hash_info": { "type": ["@", "rules", "CC/test", "test"] , "name": ["hash_info"] , "srcs": ["hash_info.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/crypto", "hash_info"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "crypto"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["crypto"] , "deps": ["hash_function", "hash_info", "hasher"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/crypto/hash_function.test.cpp000066400000000000000000000050761516554100600304350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hash_function.hpp" #include #include // std::move #include "catch2/catch_test_macros.hpp" #include "src/buildtool/crypto/hasher.hpp" TEST_CASE("Hash Function", "[crypto]") { std::string bytes{"test"}; SECTION("GitSHA1") { HashFunction const hash_function{HashFunction::Type::GitSHA1}; // same as: echo -n test | sha1sum CHECK(hash_function.PlainHashData(bytes).HexString() == "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); // same as: echo -n test | git hash-object --stdin CHECK(hash_function.HashBlobData(bytes).HexString() == "30d74d258442c7c65512eafab474568dd706c430"); // same as: echo -n test | git hash-object -t "tree" --stdin --literally CHECK(hash_function.HashTreeData(bytes).HexString() == "5f0ecc1a989593005e80f457446133250fcc43cc"); auto hasher = hash_function.MakeHasher(); hasher.Update(bytes); CHECK(std::move(hasher).Finalize().HexString() == // NOLINT "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); } SECTION("PlainSHA256") { HashFunction const hash_function{HashFunction::Type::PlainSHA256}; // all same as: echo -n test | sha256sum CHECK( hash_function.PlainHashData(bytes).HexString() == "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); CHECK( hash_function.HashBlobData(bytes).HexString() == "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); CHECK( hash_function.HashTreeData(bytes).HexString() == "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); auto hasher = hash_function.MakeHasher(); hasher.Update(bytes); CHECK( std::move(hasher).Finalize().HexString() == // NOLINT "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/crypto/hash_info.test.cpp000066400000000000000000000072601516554100600275400ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hash_info.hpp" #include "catch2/catch_test_macros.hpp" #include "src/buildtool/crypto/hash_function.hpp" inline constexpr auto kValidGitSHA1 = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"; inline constexpr auto kInvalidGitSHA1 = "e69de29bb2d1d6434b8b29ae775ad8c2e48c539z"; inline constexpr auto kValidPlainSHA256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; inline constexpr auto kInvalidPlainSHA256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7az"; TEST_CASE("Empty HashInfo", "[crypto]") { HashInfo info; CHECK_FALSE(info.Hash().empty()); CHECK(info.HashType() == HashFunction::Type::GitSHA1); CHECK(info.IsTree() == false); } TEST_CASE("Native HashInfo", "[crypto]") { SECTION("Valid hash") { // Valid hex string of valid length CHECK(HashInfo::Create(HashFunction::Type::GitSHA1, kValidGitSHA1, /*is_tree=*/false)); CHECK(HashInfo::Create(HashFunction::Type::GitSHA1, kValidGitSHA1, /*is_tree=*/true)); } SECTION("Invalid hash") { // Invalid hex string (contains z) CHECK_FALSE(HashInfo::Create(HashFunction::Type::GitSHA1, kInvalidGitSHA1, /*is_tree=*/false)); CHECK_FALSE(HashInfo::Create(HashFunction::Type::GitSHA1, kInvalidGitSHA1, /*is_tree=*/true)); // Valid hex string, but wrong length CHECK_FALSE(HashInfo::Create(HashFunction::Type::GitSHA1, kValidPlainSHA256, /*is_tree=*/false)); CHECK_FALSE(HashInfo::Create(HashFunction::Type::GitSHA1, kValidPlainSHA256, /*is_tree=*/true)); } } TEST_CASE("Compatible HashInfo", "[crypto]") { SECTION("Valid hash") { // Valid hex string of valid length, not a tree CHECK(HashInfo::Create(HashFunction::Type::PlainSHA256, kValidPlainSHA256, /*is_tree=*/false)); } SECTION("No trees") { // Valid hex string of valid length, a tree that is not allowed in // the compatible mode CHECK_FALSE(HashInfo::Create(HashFunction::Type::PlainSHA256, kValidPlainSHA256, /*is_tree=*/true)); } SECTION("Invalid hash") { // Invalid hex string (contains z) CHECK_FALSE(HashInfo::Create(HashFunction::Type::PlainSHA256, kInvalidPlainSHA256, /*is_tree=*/false)); // Valid hex string, but wrong length CHECK_FALSE(HashInfo::Create(HashFunction::Type::PlainSHA256, kValidGitSHA1, /*is_tree=*/false)); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/crypto/hasher.test.cpp000066400000000000000000000036561516554100600270610ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/crypto/hasher.hpp" #include #include #include // std::move #include "catch2/catch_test_macros.hpp" template void test_increment_hash(std::string const& bytes, std::string const& result) { auto hasher = Hasher::Create(kType); REQUIRE(hasher.has_value()); hasher->Update(bytes.substr(0, bytes.size() / 2)); hasher->Update(bytes.substr(bytes.size() / 2)); auto digest = std::move(*hasher).Finalize(); CHECK(digest.HexString() == result); } TEST_CASE("Hasher", "[crypto]") { std::string bytes{"test"}; SECTION("SHA-1") { // same as: echo -n test | sha1sum test_increment_hash( bytes, "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); } SECTION("SHA-256") { // same as: echo -n test | sha256sum test_increment_hash( bytes, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); } SECTION("SHA-512") { // same as: echo -n test | sha512sum test_increment_hash( bytes, "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27a" "c185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/000077500000000000000000000000001516554100600254275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/TARGETS000066400000000000000000000007711516554100600264700ustar00rootroot00000000000000{ "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "data/executable_file" , "data/non_executable_file" , "data/subdir1/file1" , "data/subdir1/subdir2/file2" ] , "stage": ["test", "buildtool", "execution_api"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["execution_api"] , "deps": [ ["./", "bazel", "TESTS"] , ["./", "common", "TESTS"] , ["./", "execution_service", "TESTS"] , ["./", "local", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/000077500000000000000000000000001516554100600265245ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/TARGETS000066400000000000000000000154611516554100600275670ustar00rootroot00000000000000{ "cas_client": { "type": ["utils/remote_execution", "CC test"] , "name": ["cas_client"] , "srcs": ["bazel_cas_client.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] , ["utils", "test_remote_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "execution_client": { "type": ["utils/remote_execution", "CC test"] , "name": ["execution_client"] , "srcs": ["bazel_execution_client.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "bazel_digest_factory"] , ["@", "src", "src/buildtool/common", "bazel_types"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/bazel_msg", "execution_config"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["utils", "catch-main-remote-execution"] , ["utils", "execution_bazel"] , ["utils", "test_auth_config"] , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "bytestream_client": { "type": ["utils/remote_execution", "CC test"] , "name": ["bytestream_client"] , "srcs": ["bytestream_client.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "grpc", "", "grpc"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/common", "ids"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "execution_bazel"] , ["utils", "test_auth_config"] , ["utils", "test_remote_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "network": { "type": ["utils/remote_execution", "CC test"] , "name": ["network"] , "srcs": ["bazel_network.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "grpc", "", "grpc"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/crypto", "hash_info"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "execution_bazel"] , ["utils", "test_auth_config"] , ["utils", "test_remote_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "msg_factory": { "type": ["@", "rules", "CC/test", "test"] , "name": ["msg_factory"] , "srcs": ["bazel_msg_factory.test.cpp"] , "data": [["buildtool/storage", "test_data"]] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/bazel_msg" , "bazel_msg_factory" ] , ["@", "src", "src/buildtool/execution_api/bazel_msg", "directory_tree"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "bazel_api": { "type": ["utils/remote_execution", "CC test"] , "name": ["bazel_api"] , "srcs": ["bazel_api.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["buildtool/execution_api/common", "api_test"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] , ["utils", "test_remote_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["bazel"] , "deps": [ "bazel_api" , "bytestream_client" , "cas_client" , "execution_client" , "msg_factory" , "network" ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/bazel_api.test.cpp000066400000000000000000000215211516554100600321350ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/buildtool/execution_api/common/api_test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" namespace { class FactoryApi final { public: explicit FactoryApi( gsl::not_null const& server_address, gsl::not_null const& auth, HashFunction hash_function, TmpDir::Ptr temp_space) noexcept : address_{*server_address}, auth_{*auth}, hash_function_{hash_function}, temp_space_{std::move(temp_space)} {} [[nodiscard]] auto operator()() const -> IExecutionApi::Ptr { static RetryConfig retry_config{}; // default retry config return IExecutionApi::Ptr{new BazelApi{"remote-execution", address_.host, address_.port, &auth_, &retry_config, {}, hash_function_, temp_space_}}; } private: ServerAddress const& address_; Auth const& auth_; HashFunction const hash_function_; TmpDir::Ptr temp_space_; }; } // namespace TEST_CASE("BazelAPI: No input, no output", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestNoInputNoOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: No input, create output", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestNoInputCreateOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: One input copied to output", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestOneInputCopiedToOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Non-zero exit code, create output", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestNonZeroExitCodeCreateOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Retrieve two identical trees to path", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveTwoIdenticalTreesToPath( api_factory, remote_config->platform_properties, "two_trees"); } TEST_CASE("BazelAPI: Retrieve file and symlink with same content to path", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveFileAndSymlinkWithSameContentToPath( api_factory, remote_config->platform_properties, "file_and_symlink"); } TEST_CASE("BazelAPI: Retrieve mixed blobs and trees", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveMixedBlobsAndTrees( api_factory, remote_config->platform_properties, "blobs_and_trees"); } TEST_CASE("BazelAPI: Create directory prior to execution", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestCreateDirPriorToExecution(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Collect file and directory symlinks", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestSymlinkCollection(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Run in different output path modes", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth); FactoryApi api_factory{ &*remote_config->remote_address, &*auth, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; TestOutputPathModes(api_factory, remote_config->platform_properties); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/bazel_cas_client.test.cpp000066400000000000000000000067361516554100600335030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" TEST_CASE("Bazel internals: CAS Client", "[execution_api]") { std::string instance_name{"remote-execution"}; std::string content("test"); auto const storage_config = TestStorageConfig::Create(); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); // Create CAS client auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); RetryConfig retry_config{}; // default retry config BazelCapabilitiesClient capabilities(remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config); BazelCasClient cas_client( remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, &capabilities, storage_config.Get().CreateTypedTmpDir("test_space")); SECTION("Valid digest and blob") { // Valid blob auto const blob = ArtifactBlob::FromMemory( storage_config.Get().hash_function, ObjectType::File, content); REQUIRE(blob.has_value()); // Search blob via digest auto digests = cas_client.FindMissingBlobs(instance_name, {blob->GetDigest()}); CHECK(digests.size() <= 1); if (not digests.empty()) { // Upload blob, if not found CHECK(cas_client.BatchUpdateBlobs(instance_name, {*blob}) == 1U); } // Read blob auto blobs = cas_client.BatchReadBlobs(instance_name, {blob->GetDigest()}); REQUIRE(blobs.size() == 1); CHECK(blobs.begin()->GetDigest() == blob->GetDigest()); auto const read_content = blobs.begin()->ReadContent(); CHECK(read_content != nullptr); CHECK(*read_content == content); } } bazel_execution_client.test.cpp000066400000000000000000000145271516554100600346560ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/bazel_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/remote_execution/bazel_action_creator.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" TEST_CASE("Bazel internals: Execution Client", "[execution_api]") { std::string instance_name{"remote-execution"}; std::string content("test"); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto test_digest = BazelDigestFactory::HashDataAs( hash_function, content); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); RetryConfig retry_config{}; // default retry config BazelExecutionClient execution_client(remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config); ExecutionConfiguration config; config.skip_cache_lookup = false; SECTION("Immediate execution and response") { auto action_immediate = CreateAction(instance_name, {"echo", "-n", content}, {}, remote_config->platform_properties, hash_function); REQUIRE(action_immediate); auto response = execution_client.Execute( instance_name, *action_immediate, config, true); REQUIRE(response.state == BazelExecutionClient::ExecutionResponse::State::Finished); REQUIRE(response.output); CHECK(response.output->action_result.stdout_digest().hash() == test_digest.hash()); } SECTION("Delayed execution") { auto action_delayed = CreateAction(instance_name, {"sh", "-c", "sleep 1s; echo -n test"}, {}, remote_config->platform_properties, hash_function); SECTION("Blocking, immediately obtain result") { auto response = execution_client.Execute( instance_name, *action_delayed, config, true); REQUIRE(response.state == BazelExecutionClient::ExecutionResponse::State::Finished); REQUIRE(response.output); CHECK(response.output->action_result.stdout_digest().hash() == test_digest.hash()); } SECTION("Non-blocking, obtain result later") { // note that the boolean false means do not wait for the stream to // become available, and it has nothing to do with waiting until the // action completes. This is WaitExecution's job :) auto response = execution_client.Execute( instance_name, *action_delayed, config, false); REQUIRE(response.state == BazelExecutionClient::ExecutionResponse::State::Ongoing); response = execution_client.WaitExecution(response.execution_handle); REQUIRE(response.output); CHECK(response.output->action_result.stdout_digest().hash() == test_digest.hash()); } } } TEST_CASE("Bazel internals: Execution Client using env variables", "[execution_api]") { std::string instance_name{"remote-execution"}; std::string content("contents of env variable"); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto test_digest = BazelDigestFactory::HashDataAs( hash_function, content); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); RetryConfig retry_config{}; // default retry config BazelExecutionClient execution_client(remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config); ExecutionConfiguration config; config.skip_cache_lookup = false; auto action = CreateAction(instance_name, {"/bin/sh", "-c", "set -e\necho -n ${MYTESTVAR}"}, {{"MYTESTVAR", content}}, remote_config->platform_properties, hash_function); REQUIRE(action); auto response = execution_client.Execute(instance_name, *action, config, true); REQUIRE(response.state == BazelExecutionClient::ExecutionResponse::State::Finished); REQUIRE(response.output); CHECK(response.output->action_result.stdout_digest().hash() == test_digest.hash()); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/bazel_msg_factory.test.cpp000066400000000000000000000136071516554100600337070ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" namespace { /// \brief Create a blob from the content found in file or symlink pointed to by /// given path. [[nodiscard]] inline auto CreateBlobFromPath( std::filesystem::path const& fpath, HashFunction hash_function) noexcept -> std::optional { auto const type = FileSystemManager::Type(fpath, /*allow_upwards=*/true); if (not type) { return std::nullopt; } auto content = FileSystemManager::ReadContentAtPath(fpath, *type); if (not content.has_value()) { return std::nullopt; } auto blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, *std::move(content)); if (not blob.has_value()) { return std::nullopt; } return *std::move(blob); } } // namespace TEST_CASE("Bazel internals: MessageFactory", "[execution_api]") { std::filesystem::path workspace{"test/buildtool/storage/data"}; std::filesystem::path subdir1 = workspace / "subdir1"; std::filesystem::path subdir2 = subdir1 / "subdir2"; std::filesystem::path file1 = subdir1 / "file1"; std::filesystem::path file2 = subdir2 / "file2"; // create a symlink std::filesystem::path link = subdir1 / "link"; REQUIRE(FileSystemManager::CreateSymlink("file1", link)); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; // create the corresponding blobs auto file1_blob = CreateBlobFromPath(file1, hash_function); auto file2_blob = CreateBlobFromPath(file2, hash_function); auto link_blob = CreateBlobFromPath(link, hash_function); CHECK(file1_blob); CHECK(file2_blob); CHECK(link_blob); // both files are the same and should result in identical blobs CHECK(*file1_blob->ReadContent() == *file2_blob->ReadContent()); CHECK(file1_blob->GetDigest().hash() == file2_blob->GetDigest().hash()); CHECK(file1_blob->GetDigest().size() == file2_blob->GetDigest().size()); // create known artifacts auto artifact1_opt = ArtifactDescription::CreateKnown( file1_blob->GetDigest(), ObjectType::File) .ToArtifact(); auto artifact1 = DependencyGraph::ArtifactNode{std::move(artifact1_opt)}; auto artifact2_opt = ArtifactDescription::CreateKnown( file2_blob->GetDigest(), ObjectType::File) .ToArtifact(); auto artifact2 = DependencyGraph::ArtifactNode{std::move(artifact2_opt)}; auto artifact3_opt = ArtifactDescription::CreateKnown( link_blob->GetDigest(), ObjectType::Symlink) .ToArtifact(); auto artifact3 = DependencyGraph::ArtifactNode{std::move(artifact3_opt)}; // create directory tree auto tree = DirectoryTree::FromNamedArtifacts({{file1.string(), &artifact1}, {file2.string(), &artifact2}, {link.string(), &artifact3}}); CHECK(tree.has_value()); // a mapping between digests and content is needed; usually via a concrete // API one gets this content either locally or from the network std::unordered_map fake_cas{ {file1_blob->GetDigest(), file1}, {file2_blob->GetDigest(), file2}, {link_blob->GetDigest(), link}}; // create blobs via tree std::unordered_set blobs{}; REQUIRE(BazelMsgFactory::CreateDirectoryDigestFromTree( *tree, [&fake_cas](std::vector const& digests, std::vector* targets) { targets->reserve(digests.size()); for (auto const& digest : digests) { REQUIRE(fake_cas.contains(digest)); auto fpath = fake_cas[digest]; if (FileSystemManager::IsNonUpwardsSymlink( fpath, /*non_strict=*/true)) { auto content = FileSystemManager::ReadSymlink(fpath); REQUIRE(content); targets->emplace_back(*content); } else { auto content = FileSystemManager::ReadFile(fpath); REQUIRE(content); targets->emplace_back(*content); } } }, [&blobs](ArtifactBlob&& blob) { blobs.emplace(std::move(blob)); return true; })); // TODO(aehlig): also check total number of DirectoryNode blobs in container } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/bazel_network.test.cpp000066400000000000000000000145301516554100600330570ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" constexpr std::size_t kLargeSize = GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH + 1; TEST_CASE("Bazel network: write/read blobs", "[execution_api]") { std::string instance_name{"remote-execution"}; auto const storage_config = TestStorageConfig::Create(); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); RetryConfig retry_config{}; // default retry config HashFunction const hash_function = storage_config.Get().hash_function; auto network = BazelNetwork{instance_name, remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, {}, hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; std::string content_foo("foo"); std::string content_bar("bar"); std::string content_baz(kLargeSize, 'x'); // single larger blob auto const foo = ArtifactBlob::FromMemory(hash_function, ObjectType::File, content_foo); REQUIRE(foo.has_value()); auto const bar = ArtifactBlob::FromMemory(hash_function, ObjectType::File, content_bar); REQUIRE(bar.has_value()); auto const baz = ArtifactBlob::FromMemory(hash_function, ObjectType::File, content_baz); REQUIRE(baz.has_value()); // Search blobs via digest REQUIRE(network.UploadBlobs({*foo, *bar, *baz})); // Read blobs in order std::vector to_read{foo->GetDigest(), bar->GetDigest(), baz->GetDigest(), bar->GetDigest(), foo->GetDigest()}; std::vector const blobs = network.CreateReader().ReadOrdered(to_read); // Check order maintained REQUIRE(blobs.size() == 5); for (auto const& blob : blobs) { REQUIRE(blob.ReadContent() != nullptr); } CHECK(*blobs[0].ReadContent() == content_foo); CHECK(*blobs[1].ReadContent() == content_bar); CHECK(*blobs[2].ReadContent() == content_baz); CHECK(*blobs[3].ReadContent() == content_bar); CHECK(*blobs[4].ReadContent() == content_foo); } TEST_CASE("Bazel network: read blobs with unknown size", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); HashFunction const hash_function = storage_config.Get().hash_function; if (not ProtocolTraits::IsNative(hash_function.GetType())) { // only supported in native mode return; } std::string instance_name{"remote-execution"}; auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); RetryConfig retry_config{}; // default retry config auto network = BazelNetwork{instance_name, remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, {}, hash_function, storage_config.Get().CreateTypedTmpDir("test_space")}; std::string content_foo("foo"); std::string content_bar(kLargeSize, 'x'); // single larger blob // Create and upload blobs: { auto const foo_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, content_foo); REQUIRE(foo_blob.has_value()); auto const bar_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, content_bar); REQUIRE(bar_blob.has_value()); REQUIRE(network.UploadBlobs({*foo_blob, *bar_blob})); } // Create digests of unknown size: auto const foo_digest = ArtifactDigest{ HashInfo::HashData(hash_function, content_foo, /*is_tree=*/false), /*size_unknown=*/0}; auto const bar_digest = ArtifactDigest{ HashInfo::HashData(hash_function, content_bar, /*is_tree=*/false), /*size_unknown=*/0}; // Read blobs std::vector const blobs = network.CreateReader().ReadOrdered({foo_digest, bar_digest}); // Check order maintained REQUIRE(blobs.size() == 2); for (auto const& blob : blobs) { REQUIRE(blob.ReadContent() != nullptr); } CHECK(*blobs[0].ReadContent() == content_foo); CHECK(*blobs[1].ReadContent() == content_bar); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/bazel/bytestream_client.test.cpp000066400000000000000000000102641516554100600337260ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/remote/bazel/bytestream_client.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" TEST_CASE("ByteStream Client: Transfer single blob", "[execution_api]") { auto storage_config = TestStorageConfig::Create(); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto stream = ByteStreamClient{remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config}; HashFunction const hash_function = storage_config.Get().hash_function; auto const temp_space = storage_config.Get().CreateTypedTmpDir("space"); SECTION("Upload small blob") { std::string instance_name{"remote-execution"}; std::string content("foobar"); // digest of "foobar" auto const blob = ArtifactBlob::FromMemory(hash_function, ObjectType::File, content); REQUIRE(blob.has_value()); CHECK(stream.Write(instance_name, *blob)); auto const downloaded_blob = stream.Read(instance_name, blob->GetDigest(), temp_space); REQUIRE(downloaded_blob.has_value()); auto const downloaded_content = downloaded_blob->ReadContent(); REQUIRE(downloaded_content != nullptr); CHECK(*downloaded_content == content); } SECTION("Upload large blob") { static constexpr std::size_t kLargeSize = GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH + 1; std::string instance_name{"remote-execution"}; std::string content(kLargeSize, '\0'); for (std::size_t i{}; i < content.size(); ++i) { content[i] = instance_name[i % instance_name.size()]; } // digest of "instance_nameinstance_nameinstance_..." auto const blob = ArtifactBlob::FromMemory(hash_function, ObjectType::File, content); REQUIRE(blob.has_value()); CHECK(stream.Write(instance_name, *blob)); SECTION("Download large blob") { auto const downloaded_blob = stream.Read(instance_name, blob->GetDigest(), temp_space); REQUIRE(downloaded_blob.has_value()); auto const downloaded_content = downloaded_blob->ReadContent(); REQUIRE(downloaded_content != nullptr); CHECK(*downloaded_content == content); } SECTION("Incrementally download large blob") { auto reader = stream.IncrementalRead(instance_name, blob->GetDigest()); std::string data{}; auto chunk = reader.Next(); while (chunk and not chunk->empty()) { data.append(chunk->begin(), chunk->end()); chunk = reader.Next(); } CHECK(chunk); CHECK(data == content); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/common/000077500000000000000000000000001516554100600267175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/common/TARGETS000066400000000000000000000066151516554100600277630ustar00rootroot00000000000000{ "api_test": { "type": ["@", "rules", "CC", "library"] , "name": ["api_test"] , "hdrs": ["api_test.hpp"] , "deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "execution_api", "common"] } , "bytestream_utils": { "type": ["@", "rules", "CC/test", "test"] , "name": ["bytestream_utils"] , "srcs": ["bytestream_utils.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "bytestream_utils"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/common", "ids"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "execution_api", "common"] } , "tree_rehashing": { "type": ["@", "rules", "CC/test", "test"] , "name": ["tree_rehashing"] , "srcs": ["tree_rehashing.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/bazel_msg" , "bazel_msg_factory" ] , ["@", "src", "src/buildtool/execution_api/common", "api_bundle"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/execution_api/utils", "rehash_utils"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "large_object_utils"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "common"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["common"] , "deps": ["bytestream_utils", "tree_rehashing"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/common/api_test.hpp000066400000000000000000001054251516554100600312470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_API_COMMON_API_TEST_HPP #define INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_API_COMMON_API_TEST_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/catch_tostring.hpp" #include "catch2/generators/catch_generators_all.hpp" #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" using ApiFactory = std::function; using ExecProps = std::map; [[nodiscard]] static inline auto CreateLocalExecConfig() noexcept -> LocalExecutionConfig { std::vector launcher{"env"}; auto* env_path = std::getenv("PATH"); if (env_path != nullptr) { launcher.emplace_back(std::string{"PATH="} + std::string{env_path}); } else { launcher.emplace_back("PATH=/bin:/usr/bin"); } LocalExecutionConfig::Builder builder; if (auto config = builder.SetLauncher(std::move(launcher)).Build()) { return *std::move(config); } Logger::Log(LogLevel::Error, "Failure setting the local launcher."); std::exit(EXIT_FAILURE); } [[nodiscard]] static inline auto GetTestDir(std::string const& test_name) -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return std::filesystem::path{tmp_dir} / test_name; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/execution_api" / test_name; } [[nodiscard]] static inline auto TestNoInputNoOutput( ApiFactory const& api_factory, ExecProps const& props, bool is_hermetic = false) { std::string test_content("test"); auto api = api_factory(); auto action = api->CreateAction(*api->UploadTree({}), {"echo", "-n", test_content}, "", {}, {}, {}, props); SECTION("Cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto response = action->Execute(); REQUIRE(response); // verify result CHECK(response->HasStdOut()); CHECK(response->StdOut() == test_content); if (is_hermetic) { CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto response = action->Execute(); REQUIRE(response); // verify result CHECK(response->HasStdOut()); CHECK(response->StdOut() == test_content); CHECK(response->IsCached()); } } } SECTION("Do not cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); // run execution auto response = action->Execute(); REQUIRE(response); // verify result CHECK(response->HasStdOut()); CHECK(response->StdOut() == test_content); CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto response = action->Execute(); REQUIRE(response); // verify result CHECK(response->HasStdOut()); CHECK(response->StdOut() == test_content); CHECK(not response->IsCached()); } } } [[nodiscard]] static inline auto TestNoInputCreateOutput( ApiFactory const& api_factory, ExecProps const& props, bool is_hermetic = false) { std::string test_content("test"); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto test_digest = ArtifactDigestFactory::HashDataAs( hash_function, test_content); std::string output_path{"output_file"}; auto api = api_factory(); auto action = api->CreateAction( *api->UploadTree({}), {"/bin/sh", "-c", "set -e\necho -n " + test_content + " > " + output_path}, "", {output_path}, {}, {}, props); SECTION("Cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); if (is_hermetic) { CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(response->IsCached()); } } } SECTION("Do not cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); } } } [[nodiscard]] static inline auto TestOneInputCopiedToOutput( ApiFactory const& api_factory, ExecProps const& props, bool is_hermetic = false) { std::string test_content("test"); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto const test_blob = ArtifactBlob::FromMemory(hash_function, ObjectType::File, test_content); REQUIRE(test_blob.has_value()); auto input_artifact_opt = ArtifactDescription::CreateKnown( test_blob->GetDigest(), ObjectType::File) .ToArtifact(); auto input_artifact = DependencyGraph::ArtifactNode{std::move(input_artifact_opt)}; std::string input_path{"dir/subdir/input"}; std::string output_path{"output_file"}; auto api = api_factory(); CHECK(api->Upload({*test_blob}, false)); auto action = api->CreateAction(*api->UploadTree({{input_path, &input_artifact}}), {"cp", input_path, output_path}, "", {output_path}, {}, {}, props); SECTION("Cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); if (is_hermetic) { CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); CHECK(response->IsCached()); } } } SECTION("Do not cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); CHECK(not response->IsCached()); } } } [[nodiscard]] static inline auto TestNonZeroExitCodeCreateOutput( ApiFactory const& api_factory, ExecProps const& props) { std::string test_content("test"); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto test_digest = ArtifactDigestFactory::HashDataAs( hash_function, test_content); std::string output_path{"output_file"}; auto api = api_factory(); auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", "set -e\necho -n " + test_content + " > " + output_path + "\nexit 1\n"}, "", {output_path}, {}, {}, props); SECTION("Cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 1); auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); SECTION("Rerun execution to verify that non-zero actions are rerun") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 1); auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); } } SECTION("Do not cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 1); auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); SECTION("Rerun execution to verify non-zero actions are not cached") { // run execution auto response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 1); auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); CHECK(not response->IsCached()); } } } [[nodiscard]] static inline auto TestRetrieveTwoIdenticalTreesToPath( ApiFactory const& api_factory, ExecProps const& props, std::string const& test_name, bool is_hermetic = false) { auto api = api_factory(); auto foo_path = std::filesystem::path{"foo"} / "baz"; auto bar_path = std::filesystem::path{"bar"} / "baz"; auto make_cmd = [&](std::string const& out_dir) { return fmt::format( "set -e\nmkdir -p {0}/{1} {0}/{2}\n" "echo -n baz > {0}/{3}\necho -n baz > {0}/{4}", out_dir, foo_path.parent_path().string(), bar_path.parent_path().string(), foo_path.string(), bar_path.string()); }; auto* path = std::getenv("PATH"); std::map env{}; if (path != nullptr) { env.emplace("PATH", path); } auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", make_cmd("root")}, "", {}, {"root"}, env, props); action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 0); if (is_hermetic) { CHECK_FALSE(response->IsCached()); } auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE_FALSE(artifacts.value()->empty()); auto info = artifacts.value()->begin()->second; SECTION("retrieve via same API object") { auto out_path = GetTestDir(test_name) / "out1"; CHECK(api->RetrieveToPaths({info}, {out_path})); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsFile(out_path / bar_path)); CHECK(FileSystemManager::ReadFile(out_path / foo_path) == FileSystemManager::ReadFile(out_path / bar_path)); } SECTION("retrive from new API object but same endpoint") { auto second_api = api_factory(); auto out_path = GetTestDir(test_name) / "out2"; CHECK(second_api->RetrieveToPaths({info}, {out_path})); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsFile(out_path / bar_path)); CHECK(FileSystemManager::ReadFile(out_path / foo_path) == FileSystemManager::ReadFile(out_path / bar_path)); } } [[nodiscard]] static inline auto TestRetrieveFileAndSymlinkWithSameContentToPath(ApiFactory const& api_factory, ExecProps const& props, std::string const& test_name, bool is_hermetic = false) { auto api = api_factory(); auto foo_path = std::filesystem::path{"foo"} / "baz"; // file auto bar_path = std::filesystem::path{"bar"} / "baz"; // symlink auto make_cmd = [&](std::string const& out_dir) { return fmt::format( "set -e\nmkdir -p {0}/{1} {0}/{2}\n" "echo -n baz > {0}/{3}\nln -s baz {0}/{4}", out_dir, foo_path.parent_path().string(), bar_path.parent_path().string(), foo_path.string(), bar_path.string()); }; auto* path = std::getenv("PATH"); std::map env{}; if (path != nullptr) { env.emplace("PATH", path); } auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", make_cmd("root")}, "", {}, {"root"}, env, props); action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 0); if (is_hermetic) { CHECK_FALSE(response->IsCached()); } auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE_FALSE(artifacts.value()->empty()); auto info = artifacts.value()->begin()->second; SECTION("retrieve via same API object") { auto out_path = GetTestDir(test_name) / "out1"; CHECK(api->RetrieveToPaths({info}, {out_path})); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsNonUpwardsSymlink(out_path / bar_path)); CHECK(FileSystemManager::ReadFile(out_path / foo_path) == FileSystemManager::ReadSymlink(out_path / bar_path)); } SECTION("retrive from new API object but same endpoint") { auto second_api = api_factory(); auto out_path = GetTestDir(test_name) / "out2"; CHECK(second_api->RetrieveToPaths({info}, {out_path})); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsNonUpwardsSymlink(out_path / bar_path)); CHECK(FileSystemManager::ReadFile(out_path / foo_path) == FileSystemManager::ReadSymlink(out_path / bar_path)); } } [[nodiscard]] static inline auto TestRetrieveMixedBlobsAndTrees( ApiFactory const& api_factory, ExecProps const& props, std::string const& test_name, bool is_hermetic = false) { auto api = api_factory(); auto foo_path = std::filesystem::path{"foo"}; auto bar_path = std::filesystem::path{"subdir"} / "bar"; auto link_path = std::filesystem::path{"sym"}; auto cmd = fmt::format("set -e\nmkdir -p {}\ntouch {} {}\nln -s dummy {}", bar_path.parent_path().string(), bar_path.string(), foo_path.string(), link_path.string()); auto* path = std::getenv("PATH"); std::map env{}; if (path != nullptr) { env.emplace("PATH", path); } auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", cmd}, "", {foo_path.string(), link_path.string()}, {bar_path.parent_path().string()}, env, props); action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result CHECK(response->ExitCode() == 0); if (is_hermetic) { CHECK_FALSE(response->IsCached()); } auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE_FALSE(artifacts.value()->empty()); std::vector paths{}; std::vector infos{}; SECTION("retrieve via same API object") { auto out_path = GetTestDir(test_name) / "out1"; std::for_each(artifacts.value()->begin(), artifacts.value()->end(), [&out_path, &paths, &infos](auto const& entry) { paths.emplace_back(out_path / entry.first); infos.emplace_back(entry.second); }); CHECK(api->RetrieveToPaths(infos, paths)); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsFile(out_path / bar_path)); CHECK(FileSystemManager::IsNonUpwardsSymlink(out_path / link_path)); } SECTION("retrieve from new API object but same endpoint") { auto second_api = api_factory(); auto out_path = GetTestDir(test_name) / "out2"; std::for_each(artifacts.value()->begin(), artifacts.value()->end(), [&out_path, &paths, &infos](auto const& entry) { paths.emplace_back(out_path / entry.first); infos.emplace_back(entry.second); }); CHECK(second_api->RetrieveToPaths(infos, paths)); CHECK(FileSystemManager::IsFile(out_path / foo_path)); CHECK(FileSystemManager::IsFile(out_path / bar_path)); CHECK(FileSystemManager::IsNonUpwardsSymlink(out_path / link_path)); } } [[nodiscard]] static inline auto TestCreateDirPriorToExecution( ApiFactory const& api_factory, ExecProps const& props, bool is_hermetic = false) { auto api = api_factory(); auto output_path = std::filesystem::path{"foo/bar/baz"}; auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", fmt::format("set -e\n [ -d {} ]\n mkdir -p {}", output_path.parent_path().string(), output_path.string())}, "", {}, {output_path}, {}, props); SECTION("Cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(IsTreeObject(artifacts.value()->at(output_path).type)); if (is_hermetic) { CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(IsTreeObject(artifacts.value()->at(output_path).type)); CHECK(response->IsCached()); } } } SECTION("Do not cache execution result in action cache") { action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(IsTreeObject(artifacts.value()->at(output_path).type)); CHECK(not response->IsCached()); SECTION("Rerun execution to verify caching") { // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(IsTreeObject(artifacts.value()->at(output_path).type)); CHECK(not response->IsCached()); } } } [[nodiscard]] static inline auto TestSymlinkCollection( ApiFactory const& api_factory, ExecProps const& props) { auto api = api_factory(); SECTION("dangling") { auto action = api->CreateAction( *api->UploadTree({}), {"/bin/sh", "-c", "set -e; " "ln -s none foo; " "rm -rf bar; ln -s none bar; " "mkdir -p baz; ln -s none baz/foo; ln -s none baz/bar"}, "", {"foo"}, {"bar", "baz"}, {}, props); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains("foo")); CHECK(IsSymlinkObject(artifacts.value()->at("foo").type)); REQUIRE(artifacts.value()->contains("bar")); CHECK(IsSymlinkObject(artifacts.value()->at("bar").type)); REQUIRE(artifacts.value()->contains("baz")); CHECK(IsTreeObject(artifacts.value()->at("baz").type)); // check if bar was correctly detected as directory symlink auto* local_response = dynamic_cast(response.get()); if (local_response != nullptr) { auto dir_symlinks = local_response->DirectorySymlinks(); REQUIRE(dir_symlinks); CHECK((*dir_symlinks)->contains("bar")); } SECTION("consuming dangling symlinks") { auto dangling_symlinks_tree = artifacts.value()->at("baz"); auto consume_action = api->CreateAction(dangling_symlinks_tree.digest, {"/bin/sh", "-c", "set -e; " "[ \"$(readlink foo)\" = \"none\" ]; " "[ \"$(readlink bar)\" = \"none\" ]; " "touch success"}, "", {"success"}, {}, {}, props); auto const consume_response = consume_action->Execute(); REQUIRE(consume_response); CHECK(consume_response->ExitCode() == 0); } } SECTION("upwards") { auto action = api->CreateAction( *api->UploadTree({}), {"/bin/sh", "-c", "set -e; " "ln -s ../foo foo; " "rm -rf bar; ln -s /bar bar; " "mkdir -p baz; ln -s ../foo baz/foo; ln -s /bar baz/bar"}, "", {"foo"}, {"bar", "baz"}, {}, props); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); if (ProtocolTraits::IsNative(api->GetHashType())) { // in native, no symlink is collected, as none of them is valid CHECK(artifacts.value()->empty()); return; } REQUIRE(artifacts.value()->contains("foo")); CHECK(IsSymlinkObject(artifacts.value()->at("foo").type)); REQUIRE(artifacts.value()->contains("bar")); CHECK(IsSymlinkObject(artifacts.value()->at("bar").type)); REQUIRE(artifacts.value()->contains("baz")); CHECK(IsTreeObject(artifacts.value()->at("baz").type)); // check if bar was correctly detected as directory symlink auto* local_response = dynamic_cast(response.get()); if (local_response != nullptr) { auto dir_symlinks = local_response->DirectorySymlinks(); REQUIRE(dir_symlinks); CHECK((*dir_symlinks)->contains("bar")); } SECTION("consuming upwards symlinks") { auto upwards_symlinks_tree = artifacts.value()->at("baz"); auto consume_action = api->CreateAction(upwards_symlinks_tree.digest, {"/bin/sh", "-c", "set -e; " "[ \"$(readlink foo)\" = \"../foo\" ]; " "[ \"$(readlink bar)\" = \"/bar\" ]; " "touch success"}, "", {"success"}, {}, {}, props); auto const consume_response = consume_action->Execute(); REQUIRE(consume_response); CHECK(consume_response->ExitCode() == 0); } } } [[nodiscard]] static inline auto TestOutputPathModes( ApiFactory const& api_factory, ExecProps const& props) { auto api = api_factory(); auto force_legacy = GENERATE(false, true); DYNAMIC_SECTION("Separated output paths (legacy=" << force_legacy << ")") { auto action = api->CreateAction(*api->UploadTree({}), {"/bin/sh", "-c", "set -e; touch foo; ln -s none fox; " "mkdir -p bar; rm -rf bat; ln -s none bat"}, "", {"foo", "fox"}, {"bar", "bat"}, {}, props, force_legacy); REQUIRE(action); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains("foo")); CHECK(IsFileObject(artifacts.value()->at("foo").type)); REQUIRE(artifacts.value()->contains("fox")); CHECK(IsSymlinkObject(artifacts.value()->at("fox").type)); REQUIRE(artifacts.value()->contains("bar")); CHECK(IsTreeObject(artifacts.value()->at("bar").type)); REQUIRE(artifacts.value()->contains("bat")); CHECK(IsSymlinkObject(artifacts.value()->at("bat").type)); // verify DirectorySymlinks auto* local_response = dynamic_cast(response.get()); if (local_response != nullptr) { auto dir_symlinks = local_response->DirectorySymlinks(); if (force_legacy or ProtocolTraits::IsNative(api->GetHashType())) { // in legacy mode (contains("bat")); } } } SECTION("Combined output paths") { auto const* local_api = dynamic_cast(api.get()); if (local_api == nullptr) { // skip: combined output paths are only available for local api return; } auto action = local_api->CreateAction( *local_api->UploadTree({}), {"/bin/sh", "-c", "set -e; touch foo; ln -s none fox; " "mkdir -p bar; rm -rf bat; ln -s none bat"}, "", {"foo", "fox", "bar", "bat"}, {}, props); REQUIRE(action); // run execution auto const response = action->Execute(); REQUIRE(response); // verify result auto const artifacts = response->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains("foo")); CHECK(IsFileObject(artifacts.value()->at("foo").type)); REQUIRE(artifacts.value()->contains("fox")); CHECK(IsSymlinkObject(artifacts.value()->at("fox").type)); REQUIRE(artifacts.value()->contains("bar")); CHECK(IsTreeObject(artifacts.value()->at("bar").type)); REQUIRE(artifacts.value()->contains("bat")); CHECK(IsSymlinkObject(artifacts.value()->at("bat").type)); // with combined output paths, we cannot report output dir symlinks auto* local_response = dynamic_cast(response.get()); if (local_response != nullptr) { auto dir_symlinks = local_response->DirectorySymlinks(); REQUIRE_FALSE(dir_symlinks); } } } #endif // INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_API_COMMON_API_TEST_HPP just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/common/bytestream_utils.test.cpp000066400000000000000000000152341516554100600340050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" TEST_CASE("ReadRequest", "[common]") { static constexpr auto* kEmpty = ""; static constexpr auto* kInstanceName = "instance_name"; static constexpr auto* kLongInstanceName = "multipart/instance_name"; static constexpr auto* kVeryLongInstanceName = "some/very/long/multipart/instance_name"; static constexpr auto* kInvalidInstanceName = "actions"; HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto const digest = ArtifactDigestFactory::HashDataAs( hash_function, "test_string"); std::string const request = ByteStreamUtils::ReadRequest::ToString(kInstanceName, digest); auto const parsed = ByteStreamUtils::ReadRequest::FromString(request); auto parsed_digest = parsed->GetDigest(hash_function.GetType()); REQUIRE(parsed); REQUIRE(parsed_digest); CHECK(parsed->GetInstanceName() == kInstanceName); CHECK(parsed_digest.value() == digest); std::string const request_empty = ByteStreamUtils::ReadRequest::ToString(kEmpty, digest); auto const parsed_empty = ByteStreamUtils::ReadRequest::FromString(request_empty); auto parsed_digest_empty = parsed_empty->GetDigest(hash_function.GetType()); REQUIRE(parsed_empty); REQUIRE(parsed_digest_empty); CHECK(parsed_empty->GetInstanceName() == kEmpty); CHECK(parsed_digest_empty.value() == digest); std::string const request_long = ByteStreamUtils::ReadRequest::ToString(kLongInstanceName, digest); auto const parsed_long = ByteStreamUtils::ReadRequest::FromString(request_long); auto parsed_digest_long = parsed_long->GetDigest(hash_function.GetType()); REQUIRE(parsed_long); REQUIRE(parsed_digest_long); CHECK(parsed_long->GetInstanceName() == kLongInstanceName); CHECK(parsed_digest_long.value() == digest); std::string const request_very_long = ByteStreamUtils::ReadRequest::ToString(kVeryLongInstanceName, digest); auto const parsed_very_long = ByteStreamUtils::ReadRequest::FromString(request_very_long); auto parsed_digest_very_long = parsed_very_long->GetDigest(hash_function.GetType()); REQUIRE(parsed_very_long); REQUIRE(parsed_digest_very_long); CHECK(parsed_very_long->GetInstanceName() == kVeryLongInstanceName); CHECK(parsed_digest_long.value() == digest); std::string const request_invalid = ByteStreamUtils::ReadRequest::ToString(kInvalidInstanceName, digest); auto const parsed_invalid = ByteStreamUtils::ReadRequest::FromString(request_invalid); CHECK(parsed_invalid == std::nullopt); } TEST_CASE("WriteRequest", "[common]") { static constexpr auto* kEmpty = ""; static constexpr auto* kInstanceName = "instance_name"; static constexpr auto* kLongInstanceName = "multipart/instance_name"; static constexpr auto* kVeryLongInstanceName = "some/very/long/multipart/instance_name"; static constexpr auto* kInvalidInstanceName = "actions"; HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto id = CreateProcessUniqueId(); REQUIRE(id); std::string const uuid = CreateUUIDVersion4(*id); auto const digest = ArtifactDigestFactory::HashDataAs( hash_function, "test_string"); std::string const request = ByteStreamUtils::WriteRequest::ToString(kInstanceName, uuid, digest); auto const parsed = ByteStreamUtils::WriteRequest::FromString(request); auto parsed_digest = parsed->GetDigest(hash_function.GetType()); REQUIRE(parsed); REQUIRE(parsed_digest.has_value()); CHECK(parsed->GetInstanceName() == kInstanceName); CHECK(parsed->GetUUID() == uuid); CHECK(parsed_digest.value() == digest); std::string const request_empty = ByteStreamUtils::WriteRequest::ToString(kEmpty, uuid, digest); auto const parsed_empty = ByteStreamUtils::WriteRequest::FromString(request_empty); auto parsed_digest_empty = parsed_empty->GetDigest(hash_function.GetType()); REQUIRE(parsed_empty); REQUIRE(parsed_digest_empty.has_value()); CHECK(parsed_empty->GetInstanceName() == kEmpty); CHECK(parsed_empty->GetUUID() == uuid); CHECK(parsed_digest_empty.value() == digest); std::string const request_long = ByteStreamUtils::WriteRequest::ToString( kLongInstanceName, uuid, digest); auto const parsed_long = ByteStreamUtils::WriteRequest::FromString(request_long); auto parsed_digest_long = parsed_long->GetDigest(hash_function.GetType()); REQUIRE(parsed_long); REQUIRE(parsed_digest_long.has_value()); CHECK(parsed_long->GetInstanceName() == kLongInstanceName); CHECK(parsed_long->GetUUID() == uuid); CHECK(parsed_digest_long.value() == digest); std::string const request_very_long = ByteStreamUtils::WriteRequest::ToString( kVeryLongInstanceName, uuid, digest); auto const parsed_very_long = ByteStreamUtils::WriteRequest::FromString(request_very_long); auto parsed_digest_very_long = parsed_very_long->GetDigest(hash_function.GetType()); REQUIRE(parsed_very_long); REQUIRE(parsed_digest_very_long.has_value()); CHECK(parsed_very_long->GetInstanceName() == kVeryLongInstanceName); CHECK(parsed_very_long->GetUUID() == uuid); CHECK(parsed_digest_very_long.value() == digest); std::string const request_invalid = ByteStreamUtils::WriteRequest::ToString( kInvalidInstanceName, uuid, digest); auto const parsed_invalid = ByteStreamUtils::WriteRequest::FromString(request_invalid); CHECK(parsed_invalid == std::nullopt); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/common/tree_rehashing.test.cpp000066400000000000000000000314241516554100600333740ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/large_objects/large_object_utils.hpp" namespace { [[nodiscard]] auto GenerateTestDirectory() -> std::optional; /// \brief Deeply hash a local tree and add it to the storage. [[nodiscard]] auto StoreHashedTree(Storage const& storage, std::filesystem::path const& path) noexcept -> std::optional; /// \brief Deeply hash a local tree, doesn't add anything to the /// storage, just calls for ArtifactDigestFactory. [[nodiscard]] auto HashTree(HashFunction::Type hash_type, std::filesystem::path const& path) noexcept -> std::optional; } // namespace TEST_CASE("Rehash tree", "[common]") { // Read storage config from the environment: auto const env_config = TestStorageConfig::Create(); // Deploy native storage: auto const native_config = StorageConfig::Builder::Rebuild(env_config.Get()) .SetHashType(HashFunction::Type::GitSHA1) .Build(); REQUIRE(native_config); // Deploy compatible storage: auto const compatible_config = StorageConfig::Builder::Rebuild(env_config.Get()) .SetHashType(HashFunction::Type::PlainSHA256) .Build(); REQUIRE(compatible_config); // Randomize test directory: auto const test_dir = GenerateTestDirectory(); REQUIRE(test_dir.has_value()); auto const& test_dir_path = (*test_dir)->GetPath(); auto const check_rehash = [&test_dir_path]( StorageConfig const& source_config, StorageConfig const& target_config) -> void { auto const source = Storage::Create(&source_config); auto const target = Storage::Create(&target_config); // Add digest to the source storage: auto const stored_digest = StoreHashedTree(source, test_dir_path); REQUIRE(stored_digest.has_value()); // calculate the "expected" after rehashing digest: auto const expected_rehashed = HashTree(target.GetHashFunction().GetType(), test_dir_path); REQUIRE(expected_rehashed.has_value()); // Rehash source digest present in the source storage and add // it to the target storage. The resulting digest must be // equal to expected_rehashed. auto const rehashed = RehashUtils::RehashDigest( {Artifact::ObjectInfo{.digest = *stored_digest, .type = ObjectType::Tree}}, source_config, target_config, /*apis=*/std::nullopt); REQUIRE(rehashed.has_value()); CHECK(rehashed->front().digest.hash() == expected_rehashed->hash()); }; SECTION("GitTree to bazel::Directory") { check_rehash(*native_config, *compatible_config); } SECTION("bazel::Directory to GitTree") { check_rehash(*compatible_config, *native_config); } // Emulating the scenario when only the top-level bazel::Directory is // available locally, and to rehash it to a git_tree, the parts must be // downloaded from the remote endpoint. // In the context of this test, "remote" is not a real remote // endpoint: it is emulated using one more Storage that is deployed in a // temporary directory. This "remote" storage contains the whole // bazel::Directory and can be passed to ApiBundle's remote field to be used // for downloading of artifacts that are unknown to the local storage. SECTION("partially available bazel::Directory") { // Provide aliases to be clear in regard of the direction of rehashing: auto const& source_config = *compatible_config; auto const& target_config = *native_config; auto const source_storage = Storage::Create(&source_config); // Deploy one more "remote" storage: auto const tmp_dir = source_config.CreateTypedTmpDir("remote"); auto const remote_config = StorageConfig::Builder{} .SetBuildRoot(tmp_dir->GetPath()) .SetHashType(source_config.hash_function.GetType()) .Build(); REQUIRE(remote_config); auto remote_storage = Storage::Create(&remote_config.value()); // Store the whole bazel::Directory to the "remote" storage: auto const stored_digest = StoreHashedTree(remote_storage, test_dir_path); REQUIRE(stored_digest.has_value()); // Get the expected result of rehashing: auto const expected_rehashed = HashTree(target_config.hash_function.GetType(), test_dir_path); REQUIRE(expected_rehashed.has_value()); // Add the top-level bazel::Directory only to the source storage: auto const top_tree_path = remote_storage.CAS().TreePath(stored_digest.value()); REQUIRE(top_tree_path); auto source_top_tree_digest = source_storage.CAS().StoreTree(*top_tree_path); REQUIRE(source_top_tree_digest.has_value()); REQUIRE(*source_top_tree_digest == *stored_digest); // Create parts of ApiBundle, taking into account that "remote" is a // LocalApi as well. LocalExecutionConfig const dump_exec_config{}; LocalContext const local_context{.exec_config = &dump_exec_config, .storage_config = &source_config, .storage = &source_storage}; LocalContext const remote_context{.exec_config = &dump_exec_config, .storage_config = &*remote_config, .storage = &remote_storage}; ApiBundle const apis{ .local = std::make_shared(&local_context), .remote = std::make_shared(&remote_context)}; // Rehash the top-level directory. This operation requires "downloading" // of unknown parts of the trees from the "remote". auto const rehashed = RehashUtils::RehashDigest( {Artifact::ObjectInfo{.digest = *stored_digest, .type = ObjectType::Tree}}, *compatible_config, *native_config, &apis); REQUIRE(rehashed.has_value()); CHECK(rehashed->front().digest.hash() == expected_rehashed->hash()); } } namespace { [[nodiscard]] auto GenerateTestDirectory() -> std::optional { auto const test_dir = FileSystemManager::GetCurrentDirectory() / "tmp"; auto head_temp_directory = TmpDir::Create(test_dir / "head_dir"); auto const head_temp_dir_path = head_temp_directory->GetPath(); // ├── exec_1 // ├── file_1 // ├── symlink_to_nested_dir_1_1 -> nested_dir_1 / nested_dir_1_1 // ├── symlink_to_nested_dir_2_1 -> nested_dir_2 / nested_dir_2_1 // ├── nested_dir_1 // │ ├── ... // │ ├── nested_dir_1_1 // │ │ └── ... // │ └── nested_dir_1_2 // │ └── ... // └── nested_dir_2 // ├── ... // ├── nested_dir_2_1 // │ └── ... // └── nested_dir_2_2 // └── ... static constexpr std::size_t kFileSize = 128; auto const file_path = head_temp_dir_path / "file_1"; if (not LargeObjectUtils::GenerateFile(file_path, kFileSize)) { return std::nullopt; } auto const exec_path = head_temp_dir_path / "exec_1"; if (not LargeObjectUtils::GenerateFile(exec_path, kFileSize, /*is_executable =*/true)) { return std::nullopt; } std::array const directories = { head_temp_dir_path / "nested_dir_1", head_temp_dir_path / "nested_dir_1" / "nested_dir_1_1", head_temp_dir_path / "nested_dir_1" / "nested_dir_1_2", head_temp_dir_path / "nested_dir_2", head_temp_dir_path / "nested_dir_2" / "nested_dir_2_1", head_temp_dir_path / "nested_dir_2" / "nested_dir_2_2"}; static constexpr std::size_t kDirEntries = 16; for (auto const& path : directories) { if (not LargeObjectUtils::GenerateDirectory(path, kDirEntries)) { return std::nullopt; } } // Create non-upwards symlinks in the top directory: if (not FileSystemManager::CreateNonUpwardsSymlink( std::filesystem::path("nested_dir_1") / "nested_dir_1_1", head_temp_dir_path / "symlink_to_nested_dir_1_1") or not FileSystemManager::CreateNonUpwardsSymlink( std::filesystem::path("nested_dir_2") / "nested_dir_2_1", head_temp_dir_path / "symlink_to_nested_dir_2_1")) { return std::nullopt; } return head_temp_directory; } [[nodiscard]] auto StoreHashedTree(Storage const& storage, std::filesystem::path const& path) noexcept -> std::optional { auto const& cas = storage.CAS(); auto store_blob = [&cas](std::filesystem::path const& path, auto is_exec) -> std::optional { return cas.StoreBlob(path, is_exec); }; auto store_tree = [&cas](std::string const& content) -> std::optional { return cas.StoreTree(content); }; auto store_symlink = [&cas](std::string const& content) -> std::optional { return cas.StoreBlob(content); }; return ProtocolTraits::IsNative(cas.GetHashFunction().GetType()) ? BazelMsgFactory::CreateGitTreeDigestFromLocalTree( path, store_blob, store_tree, store_symlink) : BazelMsgFactory::CreateDirectoryDigestFromLocalTree( path, store_blob, store_tree, store_symlink); } [[nodiscard]] auto HashTree(HashFunction::Type hash_type, std::filesystem::path const& path) noexcept -> std::optional { HashFunction const hash_function{hash_type}; auto hash_blob = [hash_function]( std::filesystem::path const& path, auto /*is_exec*/) -> std::optional { return ArtifactDigestFactory::HashFileAs( hash_function, path); }; auto hash_tree = [hash_function]( std::string const& content) -> std::optional { return ArtifactDigestFactory::HashDataAs( hash_function, content); }; auto hash_symlink = [hash_function]( std::string const& content) -> std::optional { return ArtifactDigestFactory::HashDataAs( hash_function, content); }; return ProtocolTraits::IsNative(hash_type) ? BazelMsgFactory::CreateGitTreeDigestFromLocalTree( path, hash_blob, hash_tree, hash_symlink) : BazelMsgFactory::CreateDirectoryDigestFromLocalTree( path, hash_blob, hash_tree, hash_symlink); } } // namespace just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/execution_service/000077500000000000000000000000001516554100600311525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/execution_service/TARGETS000066400000000000000000000055351516554100600322160ustar00rootroot00000000000000{ "cas_server": { "type": ["@", "rules", "CC/test", "test"] , "name": ["cas_server"] , "srcs": ["cas_server.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "bazel_digest_factory"] , ["@", "src", "src/buildtool/common", "bazel_types"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/execution_service" , "cas_server" ] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/file_system", "git_repo"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "execution_service"] } , "execution_server": { "type": ["@", "rules", "CC/test", "test"] , "name": ["execution_server"] , "srcs": ["execution_server.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "gsl", "", "gsl"] , ["@", "protoc", "", "libprotobuf"] , ["@", "src", "src/buildtool/common", "bazel_digest_factory"] , ["@", "src", "src/buildtool/common", "bazel_types"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/execution_service" , "cas_server" ] , [ "@" , "src" , "src/buildtool/execution_api/execution_service" , "execution_server" ] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/file_system", "git_repo"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["utils", "test_hash_function_type"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "execution_service"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["execution_service"] , "deps": ["cas_server", "execution_server"] } } cas_server.test.cpp000066400000000000000000000103211516554100600347060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/execution_service// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include #include #include #include #include #include // Don't include "proto" // IWYU pragma: no_include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/bazel_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { [[nodiscard]] auto Upload( gsl::not_null const& cas_server, std::string const& instance_name, bazel_re::Digest const& digest, std::string const& content) noexcept -> grpc::Status { auto request = bazel_re::BatchUpdateBlobsRequest{}; request.set_instance_name(instance_name); auto* req = request.add_requests(); req->mutable_digest()->CopyFrom(digest); req->set_data(content); auto response = bazel_re::BatchUpdateBlobsResponse{}; return cas_server->BatchUpdateBlobs(nullptr, &request, &response); } } // namespace TEST_CASE("CAS Service: upload incomplete tree", "[execution_service]") { // For compatible mode tree invariants aren't checked. if (not ProtocolTraits::IsNative(TestHashType::ReadFromEnvironment())) { return; } auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig const local_exec_config{}; // pack the local context instances to be passed LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; auto cas_server = CASServiceImpl{&local_context}; auto instance_name = std::string{"remote-execution"}; // Create an empty tree. auto empty_entries = GitRepo::tree_entries_t{}; auto empty_tree = GitRepo::CreateShallowTree(empty_entries); REQUIRE(empty_tree); auto empty_tree_digest = BazelDigestFactory::HashDataAs( storage_config.Get().hash_function, empty_tree->second); // Create a tree containing the empty tree. auto entries = GitRepo::tree_entries_t{}; entries[empty_tree->first].emplace_back("empty_tree", ObjectType::Tree); auto tree = GitRepo::CreateShallowTree(entries); REQUIRE(tree); auto tree_digest = BazelDigestFactory::HashDataAs( storage_config.Get().hash_function, tree->second); // Upload tree. The tree invariant is violated, thus, a negative answer is // expected. auto status = Upload(&cas_server, instance_name, tree_digest, tree->second); CHECK(not status.ok()); // Upload empty tree. status = Upload( &cas_server, instance_name, empty_tree_digest, empty_tree->second); CHECK(status.ok()); // Upload tree again. This time, the tree invariant is honored and a // positive answer is expected. status = Upload(&cas_server, instance_name, tree_digest, tree->second); CHECK(status.ok()); } execution_server.test.cpp000066400000000000000000000326641516554100600361610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/execution_service// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include // Don't include "proto" // IWYU pragma: no_include "google/longrunning/operations.pb.h" // IWYU pragma: no_include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "catch2/catch_test_macros.hpp" #include "catch2/catch_tostring.hpp" #include "catch2/generators/catch_generators_all.hpp" #include "fmt/core.h" #include "google/protobuf/repeated_ptr_field.h" #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { auto const kV20 = Capabilities::Version{.major = 2, .minor = 0, .patch = 0}; auto const kV21 = Capabilities::Version{.major = 2, .minor = 1, .patch = 0}; // Class to obtain a valid pointer to internal ServerWriter<...::Operation> class MockServerWriter final : public ::grpc::ServerWriterInterface<::google::longrunning::Operation> { public: MockServerWriter() = default; [[nodiscard]] auto Get() -> ::grpc::ServerWriter<::google::longrunning::Operation>* { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) return reinterpret_cast< ::grpc::ServerWriter<::google::longrunning::Operation>*>(this); } // stub implementations void SendInitialMetadata() override {} using ::grpc::internal::WriterInterface< google::longrunning::Operation>::Write; auto Write(google::longrunning::Operation const& /*msg*/, grpc::WriteOptions /*options*/) -> bool override { return true; } private: MockServerWriter(grpc::internal::Call* /*call*/, grpc::ServerContext* /*ctx*/) {} }; template [[nodiscard]] auto Upload( gsl::not_null const& cas_server, std::string const& instance_name, gsl::not_null const& storage_config, std::string const& content) noexcept -> std::optional { auto digest = BazelDigestFactory::HashDataAs( storage_config->hash_function, content); auto request = bazel_re::BatchUpdateBlobsRequest{}; request.set_instance_name(instance_name); auto* req = request.add_requests(); req->mutable_digest()->CopyFrom(digest); req->set_data(content); auto response = bazel_re::BatchUpdateBlobsResponse{}; if (cas_server->BatchUpdateBlobs(nullptr, &request, &response).ok()) { return digest; } return std::nullopt; } [[nodiscard]] auto CreateEmptyTree( gsl::not_null const& cas_server, gsl::not_null const& storage_config, std::string const& instance_name) noexcept { if (ProtocolTraits::IsNative(TestHashType::ReadFromEnvironment())) { auto empty_entries = GitRepo::tree_entries_t{}; auto empty_tree = GitRepo::CreateShallowTree(empty_entries); REQUIRE(empty_tree); auto digest = Upload( cas_server, instance_name, storage_config, empty_tree->second); REQUIRE(digest); return *digest; } auto digest = Upload(cas_server, instance_name, storage_config, bazel_re::Directory{}.SerializeAsString()); REQUIRE(digest); return *digest; } [[nodiscard]] auto Execute( gsl::not_null const& cas_server, gsl::not_null const& exec_server, gsl::not_null const& storage_config, std::string const& instance_name, bazel_re::Digest const& root_digest, std::string const& cwd, std::vector const& argv, std::vector output_files, std::vector output_dirs, std::map const& env, std::map const& properties, Capabilities::Version const& version) noexcept -> std::optional { auto get_platform = [&properties]() { auto platform = std::make_unique(); std::transform(properties.begin(), properties.end(), pb::back_inserter(platform->mutable_properties()), [](auto prop) { auto out = bazel_re::Platform_Property{}; out.set_name(prop.first); out.set_value(prop.second); return out; }); return platform.release(); }; // create command auto cmd = bazel_re::Command{}; std::copy( argv.begin(), argv.end(), pb::back_inserter(cmd.mutable_arguments())); cmd.set_working_directory(cwd); std::transform(env.begin(), env.end(), pb::back_inserter(cmd.mutable_environment_variables()), [](auto const& name_val) { auto var = bazel_re::Command_EnvironmentVariable{}; var.set_name(name_val.first); var.set_value(name_val.second); return var; }); if (version >= kV21) { auto paths = std::vector{}; paths.reserve(output_files.size() + output_dirs.size()); paths.insert(paths.end(), std::move_iterator{output_files.begin()}, std::move_iterator{output_files.end()}); paths.insert(paths.end(), std::move_iterator{output_dirs.begin()}, std::move_iterator{output_dirs.end()}); std::sort(paths.begin(), paths.end()); std::copy(paths.begin(), paths.end(), pb::back_inserter(cmd.mutable_output_paths())); } else { std::sort(output_files.begin(), output_files.end()); std::sort(output_dirs.begin(), output_dirs.end()); std::copy(output_files.begin(), output_files.end(), pb::back_inserter(cmd.mutable_output_files())); std::copy(output_dirs.begin(), output_dirs.end(), pb::back_inserter(cmd.mutable_output_directories())); } cmd.set_allocated_platform(get_platform()); auto cmd_digest = Upload( cas_server, instance_name, storage_config, cmd.SerializeAsString()); REQUIRE(cmd_digest); // create action auto action = bazel_re::Action{}; action.mutable_command_digest()->CopyFrom(*cmd_digest); action.mutable_input_root_digest()->CopyFrom(root_digest); auto action_digest = Upload( cas_server, instance_name, storage_config, action.SerializeAsString()); REQUIRE(action_digest); // create execute request auto request = bazel_re::ExecuteRequest{}; request.set_instance_name(instance_name); request.mutable_action_digest()->CopyFrom(*action_digest); // mock server-internal execute call auto writer = MockServerWriter{}; auto status = exec_server->Execute(nullptr, &request, writer.Get()); if (status.ok()) { if (auto just_digest = ArtifactDigestFactory::FromBazel( storage_config->hash_function.GetType(), *action_digest)) { return *std::move(just_digest); } } return std::nullopt; } [[nodiscard]] auto ToString(Capabilities::Version const& version) -> std::string { return fmt::format("{}.{}.{}", version.major, version.minor, version.patch); } } // namespace TEST_CASE("Execution Service: Test supported API versions", "[execution_service]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig const local_exec_config{}; // pack the local context instances to be passed LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; auto local_api = LocalApi{&local_context}; auto exec_server = ExecutionServiceImpl{&local_context, &local_api, std::nullopt}; auto cas_server = CASServiceImpl{&local_context}; auto instance_name = std::string{"remote-execution"}; auto root_digest = CreateEmptyTree(&cas_server, &storage_config.Get(), instance_name); auto env = std::map{}; if (auto const* path_var = std::getenv("PATH")) { // server executes locally, make sure it knows about PATH from TEST_ENV env.emplace("PATH", path_var); } auto version = GENERATE(kV20, kV21); DYNAMIC_SECTION("Pretend being a client using RBEv" << ToString(version)) { auto action_digest = Execute(&cas_server, &exec_server, &storage_config.Get(), instance_name, root_digest, "", {"/bin/sh", "-c", "set -e; touch foo; ln -s none fox; " "mkdir -p bar; rm -rf bat; ln -s none bat"}, {"foo", "fox"}, {"bar", "bat"}, env, {}, version); REQUIRE(action_digest); auto result = storage.ActionCache().CachedResult(*action_digest); REQUIRE(result); // check output files and directories CHECK_FALSE(result->output_files().empty()); CHECK_FALSE(result->output_directories().empty()); if (not result->output_files().empty()) { CHECK(result->output_files().begin()->path() == "foo"); } if (not result->output_directories().empty()) { CHECK(result->output_directories().begin()->path() == "bar"); } // check output symlinks if (version >= kV21) { // starting from RBEv2.1, output_symlinks must be filled CHECK(result->output_symlinks_size() == 2); if (result->output_symlinks_size() == 2) { auto paths = std::unordered_set{}; paths.reserve(2); std::transform(result->output_symlinks().begin(), result->output_symlinks().end(), std::inserter(paths, paths.end()), [](auto const& link) { return link.path(); }); CHECK(paths.contains("fox")); CHECK(paths.contains("bat")); } // separated file/dir symlinks may be reported additionally if (not result->output_file_symlinks().empty()) { CHECK(result->output_file_symlinks().begin()->path() == "fox"); } if (not result->output_directory_symlinks().empty()) { CHECK(result->output_directory_symlinks().begin()->path() == "bat"); } } else { // in legacy mode, output_symlinks must not be set... CHECK(result->output_symlinks().empty()); // ... instead, file/dir symlinks must be reported separately CHECK_FALSE(result->output_file_symlinks().empty()); CHECK_FALSE(result->output_directory_symlinks().empty()); if (not result->output_file_symlinks().empty()) { CHECK(result->output_file_symlinks().begin()->path() == "fox"); } if (not result->output_directory_symlinks().empty()) { CHECK(result->output_directory_symlinks().begin()->path() == "bat"); } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/local/000077500000000000000000000000001516554100600265215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/local/TARGETS000066400000000000000000000042331516554100600275570ustar00rootroot00000000000000{ "local_execution": { "type": ["@", "rules", "CC/test", "test"] , "name": ["local_execution"] , "srcs": ["local_execution.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "local"] } , "local_api": { "type": ["@", "rules", "CC/test", "test"] , "name": ["local_api"] , "srcs": ["local_api.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["buildtool/execution_api/common", "api_test"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "local"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["local"] , "deps": ["local_api", "local_execution"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/local/local_api.test.cpp000066400000000000000000000175371516554100600321430ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_api/local/local_api.hpp" #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/buildtool/execution_api/common/api_test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { class FactoryApi final { public: explicit FactoryApi( gsl::not_null const& local_context) noexcept : local_context_{*local_context} {} [[nodiscard]] auto operator()() const -> IExecutionApi::Ptr { return IExecutionApi::Ptr{new LocalApi{&local_context_}}; } private: LocalContext const& local_context_; }; } // namespace TEST_CASE("LocalAPI: No input, no output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestNoInputNoOutput(api_factory, {}, /*is_hermetic=*/true); } TEST_CASE("LocalAPI: No input, create output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestNoInputCreateOutput(api_factory, {}, /*is_hermetic=*/true); } TEST_CASE("LocalAPI: One input copied to output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestOneInputCopiedToOutput(api_factory, {}, /*is_hermetic=*/true); } TEST_CASE("LocalAPI: Non-zero exit code, create output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestNonZeroExitCodeCreateOutput(api_factory, {}); } TEST_CASE("LocalAPI: Retrieve two identical trees to path", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestRetrieveTwoIdenticalTreesToPath( api_factory, {}, "two_trees", /*is_hermetic=*/true); } TEST_CASE("LocalAPI: Retrieve file and symlink with same content to path", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestRetrieveFileAndSymlinkWithSameContentToPath( api_factory, {}, "file_and_symlink", /*is_hermetic=*/true); } TEST_CASE("LocalAPI: Retrieve mixed blobs and trees", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestRetrieveMixedBlobsAndTrees( api_factory, {}, "blobs_and_trees", /*is_hermetic=*/true); } TEST_CASE("LocalAPI: Create directory prior to execution", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestCreateDirPriorToExecution(api_factory, {}, /*is_hermetic=*/true); } TEST_CASE("LocalAPI: Collect file and directory symlinks", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestSymlinkCollection(api_factory, {}); } TEST_CASE("LocalAPI: Run in different output path modes", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; FactoryApi api_factory(&local_context); TestOutputPathModes(api_factory, {}); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_api/local/local_execution.test.cpp000066400000000000000000000342431516554100600333660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" namespace { constexpr bool kLegacyApi = false; // do not force legacy api logic [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/execution_api/local"; } [[nodiscard]] inline auto CreateLocalExecConfig() noexcept -> LocalExecutionConfig { std::vector launcher{"env"}; auto* env_path = std::getenv("PATH"); if (env_path != nullptr) { launcher.emplace_back(std::string{"PATH="} + std::string{env_path}); } else { launcher.emplace_back("PATH=/bin:/usr/bin"); } LocalExecutionConfig::Builder builder; if (auto config = builder.SetLauncher(std::move(launcher)).Build()) { return *std::move(config); } Logger::Log(LogLevel::Error, "Failure setting the local launcher."); std::exit(EXIT_FAILURE); } } // namespace TEST_CASE("LocalExecution: No input, no output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; auto api = LocalApi(&local_context, &repo_config); std::string test_content("test"); std::vector const cmdline = {"echo", "-n", test_content}; auto action = api.CreateAction( *api.UploadTree({}), cmdline, "", {}, {}, {}, {}, kLegacyApi); REQUIRE(action); SECTION("Cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); CHECK(output->StdOut() == test_content); output = action->Execute(nullptr); REQUIRE(output); CHECK(output->IsCached()); } SECTION("Do not cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); CHECK(output->StdOut() == test_content); // ensure result IS STILL NOT in cache output = action->Execute(nullptr); REQUIRE(output); CHECK_FALSE(output->IsCached()); } } TEST_CASE("LocalExecution: No input, no output, env variables used", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; auto api = LocalApi(&local_context, &repo_config); std::string test_content("test from env var"); std::vector const cmdline = { "/bin/sh", "-c", "set -e\necho -n ${MYCONTENT}"}; auto action = api.CreateAction(*api.UploadTree({}), cmdline, "", {}, {}, {{"MYCONTENT", test_content}}, {}, kLegacyApi); REQUIRE(action); SECTION("Cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); CHECK(output->StdOut() == test_content); // ensure result IS in cache output = action->Execute(nullptr); REQUIRE(output); CHECK(output->IsCached()); } SECTION("Do not cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); CHECK(output->StdOut() == test_content); // ensure result IS STILL NOT in cache output = action->Execute(nullptr); REQUIRE(output); CHECK_FALSE(output->IsCached()); } } TEST_CASE("LocalExecution: No input, create output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; auto api = LocalApi(&local_context, &repo_config); std::string test_content("test"); auto test_digest = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, test_content); std::string output_path{"output_file"}; std::vector const cmdline = { "/bin/sh", "-c", "set -e\necho -n " + test_content + " > " + output_path}; auto action = api.CreateAction(*api.UploadTree({}), cmdline, "", {output_path}, {}, {}, {}, kLegacyApi); REQUIRE(action); SECTION("Cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); auto const artifacts = output->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); // ensure result IS in cache output = action->Execute(nullptr); REQUIRE(output); CHECK(output->IsCached()); } SECTION("Do not cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); auto const artifacts = output->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_digest); // ensure result IS STILL NOT in cache output = action->Execute(nullptr); REQUIRE(output); CHECK_FALSE(output->IsCached()); } } TEST_CASE("LocalExecution: One input copied to output", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; auto api = LocalApi(&local_context, &repo_config); std::string test_content("test"); auto const test_blob = ArtifactBlob::FromMemory( storage_config.Get().hash_function, ObjectType::File, test_content); REQUIRE(test_blob); REQUIRE(api.Upload({*test_blob}, false)); std::string input_path{"dir/subdir/input"}; std::string output_path{"output_file"}; std::vector const cmdline = {"cp", input_path, output_path}; auto local_artifact_opt = ArtifactDescription::CreateKnown( test_blob->GetDigest(), ObjectType::File) .ToArtifact(); auto local_artifact = DependencyGraph::ArtifactNode{std::move(local_artifact_opt)}; auto action = api.CreateAction(*api.UploadTree({{input_path, &local_artifact}}), cmdline, "", {output_path}, {}, {}, {}, kLegacyApi); REQUIRE(action); SECTION("Cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); auto const artifacts = output->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); // ensure result IS in cache output = action->Execute(nullptr); REQUIRE(output); CHECK(output->IsCached()); } SECTION("Do not cache execution result in action cache") { // run execution action->SetCacheFlag(IExecutionAction::CacheFlag::DoNotCacheOutput); auto output = action->Execute(nullptr); REQUIRE(output); // verify result CHECK_FALSE(output->IsCached()); auto const artifacts = output->Artifacts(); REQUIRE(artifacts.has_value()); REQUIRE(artifacts.value()->contains(output_path)); CHECK(artifacts.value()->at(output_path).digest == test_blob->GetDigest()); // ensure result IS STILL NOT in cache output = action->Execute(nullptr); REQUIRE(output); CHECK_FALSE(output->IsCached()); } } TEST_CASE("LocalExecution: Cache failed action's result", "[execution_api]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; auto api = LocalApi(&local_context, &repo_config); auto flag = GetTestDir() / "flag"; std::vector const cmdline = { "sh", "-c", fmt::format("[ -f '{}' ]", flag.string())}; auto action = api.CreateAction( *api.UploadTree({}), cmdline, "", {}, {}, {}, {}, kLegacyApi); REQUIRE(action); action->SetCacheFlag(IExecutionAction::CacheFlag::CacheOutput); // run failed action auto failed = action->Execute(nullptr); REQUIRE(failed); CHECK_FALSE(failed->IsCached()); CHECK(failed->ExitCode() != 0); REQUIRE(FileSystemManager::CreateFile(flag)); // run success action (should rerun and overwrite) auto success = action->Execute(nullptr); REQUIRE(success); CHECK_FALSE(success->IsCached()); CHECK(success->ExitCode() == 0); // rerun success action (should be served from cache) auto cached = action->Execute(nullptr); REQUIRE(cached); CHECK(cached->IsCached()); CHECK(cached->ExitCode() == 0); CHECK(FileSystemManager::RemoveFile(flag)); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/000077500000000000000000000000001516554100600261235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/TARGETS000066400000000000000000000003741516554100600271630ustar00rootroot00000000000000{ "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["execution_engine"] , "deps": [ ["./", "dag", "TESTS"] , ["./", "executor", "TESTS"] , ["./", "traverser", "TESTS"] , ["./", "tree_operations", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/dag/000077500000000000000000000000001516554100600266565ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/dag/TARGETS000066400000000000000000000012321516554100600277100ustar00rootroot00000000000000{ "dag": { "type": ["@", "rules", "CC/test", "test"] , "name": ["dag"] , "srcs": ["dag.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["", "catch-main"] , ["utils", "container_matchers"] ] , "stage": ["test", "buildtool", "execution_engine", "dag"] } , "TESTS": {"type": ["@", "rules", "test", "suite"], "stage": ["dag"], "deps": ["dag"]} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/dag/dag.test.cpp000066400000000000000000000363521516554100600311040ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_engine/dag/dag.hpp" #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "gsl/gsl" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/identifier.hpp" #include "test/utils/container_matchers.hpp" namespace { [[nodiscard]] auto IsValidGraph(DependencyGraph const& graph) noexcept -> bool; [[nodiscard]] auto GetActionOfArtifact( DependencyGraph const& g, ActionIdentifier const& action_id) noexcept -> DependencyGraph::ActionNode const*; } // namespace /// \brief Checks that each artifact with identifier in output_ids has been /// added to the graph and that its builder action has id action_id, and that /// all outputs of actions are those the ids of which are listed in output_ids void CheckOutputNodesCorrectlyAdded( DependencyGraph const& g, ActionIdentifier const& action_id, std::vector const& output_paths) { std::vector output_ids; for (auto const& path : output_paths) { auto const output_id = ArtifactDescription::CreateAction(action_id, path).Id(); CHECK(g.ArtifactNodeWithId(output_id) != nullptr); auto const* action = GetActionOfArtifact(g, output_id); CHECK(action != nullptr); CHECK(action->Content().Id() == action_id); output_ids.push_back(output_id); } std::vector output_file_ids; for (auto const& out_file : g.ActionNodeWithId(action_id)->OutputFiles()) { output_file_ids.push_back(out_file.node->Content().Id()); } CHECK_THAT( output_file_ids, HasSameUniqueElementsAs>(output_ids)); } /// \brief Checks that the artifacts with ids in inputs_ids are in the graph and /// coincide with the action's dependencies void CheckInputNodesCorrectlyAdded( DependencyGraph const& g, ActionIdentifier const& action_id, std::vector const& input_ids) noexcept { for (auto const& input_id : input_ids) { CHECK(g.ArtifactNodeWithId(input_id) != nullptr); } std::vector dependency_ids; for (auto const& dependency : g.ActionNodeWithId(action_id)->Dependencies()) { dependency_ids.push_back(dependency.node->Content().Id()); } CHECK_THAT( dependency_ids, HasSameUniqueElementsAs>(input_ids)); } /// \brief Checks that the artifacts have been added as local artifact and their /// local path is correct void CheckLocalArtifactsCorrectlyAdded( DependencyGraph const& g, std::vector const& ids, std::vector const& paths) noexcept { REQUIRE(ids.size() == paths.size()); for (std::size_t pos = 0; pos < ids.size(); ++pos) { auto const* artifact_node = g.ArtifactNodeWithId(ids[pos]); CHECK(artifact_node != nullptr); CHECK(not artifact_node->HasBuilderAction()); CHECK(artifact_node->Content().FilePath() == paths[pos]); } } TEST_CASE("Empty Dependency Graph", "[dag]") { DependencyGraph g; CHECK(IsValidGraph(g)); } TEST_CASE("AddAction({single action, single output, no inputs})", "[dag]") { std::string const action_id = "action_id"; auto const action_description = ActionDescription{ {"out"}, {}, Action{action_id, {"touch", "out"}, {}}, {}}; DependencyGraph g; CHECK(g.AddAction(action_description)); CheckOutputNodesCorrectlyAdded(g, action_id, {"out"}); CHECK(IsValidGraph(g)); } TEST_CASE("AddAction({single action, more outputs, no inputs})", "[dag]") { std::string const action_id = "action_id"; std::vector const output_files = {"out0", "out1", "out2"}; auto const action_description = ActionDescription{ output_files, {}, Action{action_id, {"touch", "out0", "out1", "out2"}, {}}, {}}; DependencyGraph g; CHECK(g.AddAction(action_description)); CheckOutputNodesCorrectlyAdded(g, action_id, output_files); CHECK(IsValidGraph(g)); } TEST_CASE("AddAction({single action, single output, source file})", "[dag]") { using path = std::filesystem::path; std::string const action_id = "action_id"; auto const src_description = ArtifactDescription::CreateLocal(path{"main.cpp"}, "repo"); auto const& src_id = src_description.Id(); DependencyGraph g; SECTION("Input file in the same path than it is locally") { auto const action_description = ActionDescription{{"executable"}, {}, Action{action_id, {"gcc", "main.cpp"}, {}}, {{"main.cpp", src_description}}}; CHECK(g.AddAction(action_description)); } SECTION("Input file in different path from the local one") { auto const action_description = ActionDescription{{"executable"}, {}, Action{action_id, {"gcc", "src/a.cpp"}, {}}, {{"src/a.cpp", src_description}}}; CHECK(g.Add({action_description})); } CheckOutputNodesCorrectlyAdded(g, action_id, {"executable"}); CheckInputNodesCorrectlyAdded(g, action_id, {src_id}); // Now we check that the src file artifact was added with the correct path CheckLocalArtifactsCorrectlyAdded(g, {src_id}, {"main.cpp"}); // All artifacts are the source file and the executable CHECK_THAT( g.ArtifactIdentifiers(), HasSameUniqueElementsAs>( {src_id, ArtifactDescription::CreateAction(action_id, "executable").Id()})); CHECK(IsValidGraph(g)); } TEST_CASE("AddAction({single action, single output, no inputs, env_variables})", "[dag]") { std::string const action_id = "action_id"; std::string const name = "World"; DependencyGraph g; std::vector const command{ "/bin/sh", "-c", "set -e\necho 'Hello, ${NAME}' > greeting"}; std::map const env_vars{{"NAME", name}}; auto const action_description = ActionDescription{ {"greeting"}, {}, Action{action_id, command, env_vars}, {}}; CHECK(g.AddAction(action_description)); CheckOutputNodesCorrectlyAdded(g, action_id, {"greeting"}); CheckInputNodesCorrectlyAdded(g, action_id, {}); auto const* const action_node = g.ActionNodeWithId(action_id); CHECK(action_node != nullptr); CHECK(action_node->Command() == command); CHECK(action_node->Env() == env_vars); // All artifacts are the output file CHECK_THAT( g.ArtifactIdentifiers(), HasSameUniqueElementsAs>( {ArtifactDescription::CreateAction(action_id, "greeting").Id()})); CHECK(IsValidGraph(g)); } TEST_CASE("Add executable and library", "[dag]") { // Note: we don't use local bindings for members of pair as output of // functions because it seems to be problematic with Catch2's macros inside // lambdas and we want to use lambdas here to avoid repetition using path = std::filesystem::path; std::string const make_exec_id = "make_exe"; std::string const make_lib_id = "make_lib"; std::vector const make_exec_cmd = {"build", "exec"}; std::vector const make_lib_cmd = {"build", "lib.a"}; auto const main_desc = ArtifactDescription::CreateLocal(path{"main.cpp"}, ""); auto const& main_id = main_desc.Id(); auto const lib_hpp_desc = ArtifactDescription::CreateLocal(path{"lib/lib.hpp"}, ""); auto const& lib_hpp_id = lib_hpp_desc.Id(); auto const lib_cpp_desc = ArtifactDescription::CreateLocal(path{"lib/lib.cpp"}, ""); auto const& lib_cpp_id = lib_cpp_desc.Id(); auto const lib_a_desc = ArtifactDescription::CreateAction(make_lib_id, "lib.a"); auto const& lib_a_id = lib_a_desc.Id(); auto const make_exec_desc = ActionDescription{{"exec"}, {}, Action{make_exec_id, make_exec_cmd, {}}, {{"main.cpp", main_desc}, {"lib.a", lib_a_desc}}}; auto const& exec_out_id = ArtifactDescription::CreateAction(make_exec_id, "exec").Id(); auto const make_lib_desc = ActionDescription{ {"lib.a"}, {}, Action{make_lib_id, make_lib_cmd, {}}, {{"lib.hpp", lib_hpp_desc}, {"lib.cpp", lib_cpp_desc}}}; DependencyGraph g; auto check_exec = [&]() { CHECK(IsValidGraph(g)); CheckOutputNodesCorrectlyAdded(g, make_exec_id, {"exec"}); CheckInputNodesCorrectlyAdded(g, make_exec_id, {main_id, lib_a_id}); CheckLocalArtifactsCorrectlyAdded(g, {main_id}, {"main.cpp"}); CHECK_THAT(GetActionOfArtifact(g, exec_out_id)->Command(), Catch::Matchers::Equals(make_exec_cmd)); }; auto check_lib = [&]() { CHECK(IsValidGraph(g)); CheckOutputNodesCorrectlyAdded(g, make_lib_id, {"lib.a"}); CheckInputNodesCorrectlyAdded(g, make_lib_id, {lib_hpp_id, lib_cpp_id}); CheckLocalArtifactsCorrectlyAdded( g, {lib_hpp_id, lib_cpp_id}, {"lib/lib.hpp", "lib/lib.cpp"}); CHECK_THAT(GetActionOfArtifact(g, lib_a_id)->Command(), Catch::Matchers::Equals(make_lib_cmd)); }; SECTION("First exec, then lib") { CHECK(g.AddAction(make_exec_desc)); check_exec(); CHECK(g.AddAction(make_lib_desc)); check_lib(); } SECTION("First lib, then exec") { CHECK(g.AddAction(make_lib_desc)); check_lib(); CHECK(g.AddAction(make_exec_desc)); check_exec(); } SECTION("Add both with single call to `DependencyGraph::Add`") { CHECK(g.Add({make_exec_desc, make_lib_desc})); check_exec(); check_lib(); } CHECK_THAT(g.ArtifactIdentifiers(), HasSameUniqueElementsAs>( {main_id, exec_out_id, lib_a_id, lib_hpp_id, lib_cpp_id})); } // Incorrect action description tests TEST_CASE("AddAction(id, empty action description) fails", "[dag]") { DependencyGraph g; CHECK(not g.AddAction(ActionDescription{ {}, {}, Action{"id", std::vector{}, {}}, {}})); } TEST_CASE("AddAction(Empty mandatory non-empty field in action description)", "[dag]") { DependencyGraph g; CHECK(not g.AddAction(ActionDescription{ {"output0", "output1"}, {}, Action{"empty command", std::vector{}, {}}, {}})); CHECK(not g.AddAction(ActionDescription{ {}, {}, Action{"empty output", {"echo", "hello"}, {}}, {}})); } // Collision between actions tests TEST_CASE("Adding cyclic dependencies produces invalid graph", "[dag]") { std::string const action1_id = "action1"; std::string const action2_id = "action2"; auto const out1_desc = ArtifactDescription::CreateAction(action1_id, "out1"); auto const out2_desc = ArtifactDescription::CreateAction(action2_id, "out2"); auto const action1_desc = ActionDescription{{"out1"}, {}, Action{action1_id, {"touch", "out1"}, {}}, {{"dep", out2_desc}}}; auto const action2_desc = ActionDescription{{"out2"}, {}, Action{action2_id, {"touch", "out2"}, {}}, {{"dep", out1_desc}}}; DependencyGraph g; CHECK(g.Add({action1_desc, action2_desc})); CHECK(not IsValidGraph(g)); } TEST_CASE("Error when adding an action with an id already added", "[dag]") { std::string const action_id = "id"; auto const action_desc = ActionDescription{{"out"}, {}, Action{"id", {"touch", "out"}, {}}, {}}; DependencyGraph g; CHECK(g.AddAction(action_desc)); CheckOutputNodesCorrectlyAdded(g, action_id, {"out"}); CHECK(IsValidGraph(g)); CHECK(not g.AddAction(action_desc)); } TEST_CASE("Error when adding conflicting output files and directories", "[dag]") { auto const action_desc = ActionDescription{ {"out"}, {"out"}, Action{"id", {"touch", "out"}, {}}, {}}; DependencyGraph g; CHECK_FALSE(g.AddAction(action_desc)); } namespace { [[nodiscard]] auto IsValidNode( DependencyGraph::ArtifactNode const& node) noexcept -> bool { return node.Children().size() <= 1; } [[nodiscard]] auto IsValidNode(DependencyGraph::ActionNode const& node) noexcept -> bool { return not node.Parents().empty(); } template [[nodiscard]] auto IsValidNode( TNode const& node, gsl::not_null*> const& seen) -> bool { // Check the node hasn't been met yet (cycles check): // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) auto const node_id = reinterpret_cast(&node); auto [it, inserted] = seen->insert(node_id); if (not inserted) { return false; } // Check the validity of the node and all children: if (not IsValidNode(node)) { return false; } for (auto const& child : node.Children()) { if (not IsValidNode(*child, seen)) { return false; } } seen->erase(it); return true; } [[nodiscard]] auto IsValidGraph(DependencyGraph const& graph) noexcept -> bool { std::unordered_set seen{}; for (auto const& id : graph.ArtifactIdentifiers()) { DependencyGraph::ArtifactNode const* node = graph.ArtifactNodeWithId(id); try { // Check the node exists and is valid: if (node == nullptr or not IsValidNode(*node, &seen)) { return false; } } catch (...) { return false; } // There must be no pending nodes: if (not seen.empty()) { return false; } } return true; } [[nodiscard]] auto GetActionOfArtifact( DependencyGraph const& g, ActionIdentifier const& action_id) noexcept -> DependencyGraph::ActionNode const* { auto const* node = g.ArtifactNodeWithId(action_id); if (node != nullptr) { auto const& children = node->Children(); if (children.empty()) { return nullptr; } return children[0]; } return nullptr; } } // namespace just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/000077500000000000000000000000001516554100600277615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/TARGETS000066400000000000000000000140511516554100600310160ustar00rootroot00000000000000{ "executor_api_tests": { "type": ["@", "rules", "CC", "library"] , "name": ["executor_api_tests"] , "hdrs": ["executor_api.test.hpp"] , "deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common", "tree"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/remote", "context"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/execution_engine/executor", "context"] , ["@", "src", "src/buildtool/execution_engine/executor", "executor"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "test_api_bundle"] , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } , "executor": { "type": ["@", "rules", "CC/test", "test"] , "name": ["executor"] , "srcs": ["executor.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "artifact_blob"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/execution_api/remote", "context"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/execution_engine/executor", "context"] , ["@", "src", "src/buildtool/execution_engine/executor", "executor"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "test_api_bundle"] , ["utils", "test_hash_function_type"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } , "local": { "type": ["utils/remote_execution", "CC test"] , "name": ["local"] , "srcs": ["executor_api_local.test.cpp"] , "data": ["test_data"] , "private-deps": [ "executor_api_tests" , ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/storage", "storage"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } , "remote_bazel": { "type": ["utils/remote_execution", "CC test"] , "name": ["remote_bazel"] , "srcs": ["executor_api_remote_bazel.test.cpp"] , "data": ["test_data"] , "private-deps": [ "executor_api_tests" , ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/bazel_msg", "execution_config"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/storage", "config"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] , ["utils", "test_remote_config"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "data/greeter/greet.cpp" , "data/greeter/greet.hpp" , "data/greeter/greet_mod.cpp" , "data/greeter/main.cpp" , "data/hello_world/main.cpp" ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["executor"] , "deps": ["executor", "local", "remote_bazel"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/000077500000000000000000000000001516554100600306725ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/greeter/000077500000000000000000000000001516554100600323275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/greeter/greet.cpp000066400000000000000000000013571516554100600341470ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "greet.hpp" void greet(std::string const& name) { std::cout << "Hello " << name << std::endl; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/greeter/greet.hpp000066400000000000000000000012451516554100600341500ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include void greet(std::string const& name); greet_mod.cpp000066400000000000000000000014641516554100600347260ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/greeter// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "greet.hpp" // this is a modification that has no effect on the produced binary void greet(std::string const& name) { std::cout << "Hello " << name << std::endl; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/greeter/main.cpp000066400000000000000000000012731516554100600337620ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "greet.hpp" int main(void) { greet("devcloud"); return 0; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/hello_world/000077500000000000000000000000001516554100600332045ustar00rootroot00000000000000main.cpp000066400000000000000000000013211516554100600345520ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/data/hello_world// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include int main(void) { std::cout << "Hello World!" << std::endl; return 0; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/executor.test.cpp000066400000000000000000000621531516554100600333100ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_engine/executor/executor.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/executor/test_api_bundle.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" /// \brief Mockup API test config. struct TestApiConfig { struct TestArtifactConfig { bool uploads{}; bool available{}; }; struct TestExecutionConfig { bool failed{}; std::vector outputs; }; struct TestResponseConfig { bool cached{}; int exit_code{}; }; std::unordered_map artifacts; TestExecutionConfig execution; TestResponseConfig response; }; static auto NamedDigest(std::string const& str) -> ArtifactDigest { HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; return ArtifactDigestFactory::HashDataAs(hash_function, str); } /// \brief Mockup Response, stores only config and action result class TestResponse : public IExecutionResponse { public: explicit TestResponse(TestApiConfig config) noexcept : config_{std::move(config)} {} [[nodiscard]] auto Status() const noexcept -> StatusCode final { return StatusCode::Success; } [[nodiscard]] auto ExitCode() const noexcept -> int final { return config_.response.exit_code; } [[nodiscard]] auto IsCached() const noexcept -> bool final { return config_.response.cached; } [[nodiscard]] auto ExecutionDuration() noexcept -> double final { return 0; } [[nodiscard]] auto HasStdErr() const noexcept -> bool final { return true; } [[nodiscard]] auto HasStdOut() const noexcept -> bool final { return true; } [[nodiscard]] auto StdErr() noexcept -> std::string final { return {}; } [[nodiscard]] auto StdOut() noexcept -> std::string final { return {}; } [[nodiscard]] auto StdErrDigest() noexcept -> std::optional final { return std::nullopt; } [[nodiscard]] auto StdOutDigest() noexcept -> std::optional final { return std::nullopt; } [[nodiscard]] auto ActionDigest() const noexcept -> std::string const& final { static const std::string kEmptyHash; return kEmptyHash; } [[nodiscard]] auto Artifacts() noexcept -> expected, std::string> final { if (not populated_) { Populate(); } return gsl::not_null(&artifacts_); } [[nodiscard]] auto HasUpwardsSymlinks() noexcept -> expected final { return false; } private: TestApiConfig config_{}; ArtifactInfos artifacts_; bool populated_ = false; void Populate() noexcept { if (populated_) { return; } populated_ = true; ArtifactInfos artifacts{}; artifacts.reserve(config_.execution.outputs.size()); // collect files and store them for (auto const& path : config_.execution.outputs) { try { artifacts.emplace( path, Artifact::ObjectInfo{.digest = NamedDigest(path), .type = ObjectType::File}); } catch (...) { return; } } artifacts_ = std::move(artifacts); } }; /// \brief Mockup Action, stores only config class TestAction : public IExecutionAction { public: explicit TestAction(TestApiConfig config) noexcept : config_{std::move(config)} {} auto Execute(Logger const* /*unused*/) noexcept -> IExecutionResponse::Ptr final { if (config_.execution.failed) { return nullptr; } return std::make_unique(config_); } void SetCacheFlag(CacheFlag /*unused*/) noexcept final {} void SetTimeout(std::chrono::milliseconds /*unused*/) noexcept final {} private: TestApiConfig config_{}; }; /// \brief Mockup Api, use config to create action and handle artifact upload class TestApi : public IExecutionApi { public: explicit TestApi(TestApiConfig config, HashFunction::Type hash_type, TmpDir::Ptr temp_space) noexcept : config_{std::move(config)}, hash_type_{hash_type}, temp_space_{std::move(temp_space)} {} [[nodiscard]] auto CreateAction( ArtifactDigest const& /*unused*/, std::vector const& /*unused*/, std::string const& /*unused*/, std::vector const& /*unused*/, std::vector const& /*unused*/, std::map const& /*unused*/, std::map const& /*unused*/, bool /*unused*/) const noexcept -> IExecutionAction::Ptr final { return std::make_unique(config_); } [[nodiscard]] auto RetrieveToPaths( std::vector const& /*unused*/, std::vector const& /*unused*/, IExecutionApi const* /* unused */) const noexcept -> bool final { return false; // not needed by Executor } [[nodiscard]] auto RetrieveToFds( std::vector const& /*unused*/, std::vector const& /*unused*/, bool /*unused*/, IExecutionApi const* /*unused*/) const noexcept -> bool final { return false; // not needed by Executor } [[nodiscard]] auto RetrieveToCas( std::vector const& unused, IExecutionApi const& /*unused*/) const noexcept -> bool final { // Note that a false-positive "free-nonheap-object" warning is thrown by // gcc 12.2 with GNU libstdc++, if the caller passes a temporary vector // that is not used by this function. Therefore, we explicitly use this // vector here to suppress this warning. The actual value returned is // irrelevant for testing though. return unused.empty(); // not needed by Executor } [[nodiscard]] auto RetrieveToMemory( Artifact::ObjectInfo const& /*artifact_info*/) const noexcept -> std::optional override { return std::nullopt; // not needed by Executor } [[nodiscard]] auto Upload(std::unordered_set&& blobs, bool /*unused*/) const noexcept -> bool final { return std::all_of( blobs.begin(), blobs.end(), [this](auto const& blob) { // for local artifacts auto const content = blob.ReadContent(); if (content == nullptr) { return false; } auto it1 = config_.artifacts.find(*content); if (it1 != config_.artifacts.end() and it1->second.uploads) { return true; } // for known and action artifacts auto it2 = config_.artifacts.find(blob.GetDigest().hash()); return it2 != config_.artifacts.end() and it2->second.uploads; }); } [[nodiscard]] auto UploadTree( std::vector const& /*unused*/) const noexcept -> std::optional final { return ArtifactDigest{}; // not needed by Executor } [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool final { try { return config_.artifacts.at(digest.hash()).available; } catch (std::exception const& /* unused */) { return false; } } [[nodiscard]] auto GetMissingDigests( std::unordered_set const& digests) const noexcept -> std::unordered_set final { std::unordered_set result; try { for (auto const& digest : digests) { if (not config_.artifacts.at(digest.hash()).available) { result.emplace(digest); } } } catch (std::exception const& /* unused */) { return result; } return result; } [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final { return hash_type_; } [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final { return temp_space_; } private: TestApiConfig config_{}; HashFunction::Type hash_type_; TmpDir::Ptr temp_space_; }; [[nodiscard]] auto SetupConfig(std::filesystem::path const& ws) -> RepositoryConfig { auto info = RepositoryConfig::RepositoryInfo{FileRoot{ws}}; RepositoryConfig repo_config{}; repo_config.SetInfo("", std::move(info)); return repo_config; } [[nodiscard]] static auto CreateTest(gsl::not_null const& g, std::filesystem::path const& ws) -> std::pair { using path = std::filesystem::path; auto const local_cpp_desc = ArtifactDescription::CreateLocal(path{"local.cpp"}, ""); auto const known_digest = NamedDigest("known.cpp"); auto const known_cpp_desc = ArtifactDescription::CreateKnown(known_digest, ObjectType::File); auto const test_action_desc = ActionDescription{ {"output1.exe", "output2.exe"}, {}, Action{"test_action", {"cmd", "line"}, {}}, {{"local.cpp", local_cpp_desc}, {known_digest.hash(), known_cpp_desc}}}; CHECK(g->AddAction(test_action_desc)); CHECK(FileSystemManager::WriteFile("local.cpp", ws / "local.cpp")); TestApiConfig config{}; config.artifacts[NamedDigest("local.cpp").hash()].uploads = true; config.artifacts[NamedDigest("known.cpp").hash()].available = true; config.artifacts[NamedDigest("output1.exe").hash()].available = true; config.artifacts[NamedDigest("output2.exe").hash()].available = true; config.execution.failed = false; config.execution.outputs = {"output1.exe", "output2.exe"}; config.response.cached = true; config.response.exit_code = 0; return std::make_pair(config, SetupConfig(ws)); } TEST_CASE("Executor: Process artifact", "[executor]") { auto const storage_config = TestStorageConfig::Create(); std::filesystem::path workspace_path{ "test/buildtool/execution_engine/executor"}; DependencyGraph g; auto [config, repo_config] = CreateTest(&g, workspace_path); HashFunction const hash_function = storage_config.Get().hash_function; auto const local_cpp_id = ArtifactDescription::CreateLocal("local.cpp", "").Id(); auto const known_cpp_id = ArtifactDescription::CreateKnown( NamedDigest("known.cpp"), ObjectType::File) .Id(); Auth auth{}; RetryConfig retry_config{}; // default retry config RemoteExecutionConfig remote_config{}; // default remote config RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_config}; SECTION("Processing succeeds for valid config") { auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); } SECTION("Processing fails if uploading local artifact failed") { config.artifacts[NamedDigest("local.cpp").hash()].uploads = false; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(not runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); } SECTION("Processing fails if known artifact is not available") { config.artifacts[NamedDigest("known.cpp").hash()].available = false; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(known_cpp_id))); } } TEST_CASE("Executor: Process action", "[executor]") { auto const storage_config = TestStorageConfig::Create(); std::filesystem::path workspace_path{ "test/buildtool/execution_engine/executor"}; DependencyGraph g; auto [config, repo_config] = CreateTest(&g, workspace_path); HashFunction const hash_function = storage_config.Get().hash_function; auto const local_cpp_id = ArtifactDescription::CreateLocal("local.cpp", "").Id(); auto const known_cpp_id = ArtifactDescription::CreateKnown( NamedDigest("known.cpp"), ObjectType::File) .Id(); ActionIdentifier const action_id{"test_action"}; auto const output1_id = ArtifactDescription::CreateAction(action_id, "output1.exe").Id(); auto const output2_id = ArtifactDescription::CreateAction(action_id, "output2.exe").Id(); Auth auth{}; RetryConfig retry_config{}; // default retry config RemoteExecutionConfig remote_config{}; // default remote config RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &remote_config}; SECTION("Processing succeeds for valid config") { auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(runner.Process(g.ActionNodeWithId(action_id))); CHECK(runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(runner.Process(g.ArtifactNodeWithId(output2_id))); } SECTION("Processing succeeds even if result was is not cached") { config.response.cached = false; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(runner.Process(g.ActionNodeWithId(action_id))); CHECK(runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(runner.Process(g.ArtifactNodeWithId(output2_id))); } SECTION("Processing succeeds even if output is not available in CAS") { config.artifacts[NamedDigest("output2.exe").hash()].available = false; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(runner.Process(g.ActionNodeWithId(action_id))); // Note: Both output digests should be created via SaveDigests(), // but processing output2.exe fails as it is not available in CAS. CHECK(runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(output2_id))); } SECTION("Processing fails if execution failed") { config.execution.failed = true; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(not runner.Process(g.ActionNodeWithId(action_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(output2_id))); } SECTION("Processing fails if exit code is non-zero") { config.response.exit_code = 1; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(not runner.Process(g.ActionNodeWithId(action_id))); // Note: Both output digests should be missing as SaveDigests() for // both is only called if processing action succeeds. CHECK(not runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(output2_id))); } SECTION("Processing fails if any output is missing") { config.execution.outputs = {"output1.exe" /*, "output2.exe"*/}; auto api = std::make_shared( config, hash_function.GetType(), storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = &repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK(runner.Process(g.ArtifactNodeWithId(local_cpp_id))); CHECK(runner.Process(g.ArtifactNodeWithId(known_cpp_id))); CHECK(not runner.Process(g.ActionNodeWithId(action_id))); // Note: Both output digests should be missing as SaveDigests() for // both is only called if processing action succeeds. CHECK(not runner.Process(g.ArtifactNodeWithId(output1_id))); CHECK(not runner.Process(g.ArtifactNodeWithId(output2_id))); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor/executor_api.test.hpp000066400000000000000000001044711516554100600341460ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_API_TEST_HPP #define INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_API_TEST_HPP #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/execution_engine/executor/executor.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/executor/test_api_bundle.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" using ApiFactory = std::function; static inline void SetupConfig(RepositoryConfig* repo_config) { auto info = RepositoryConfig::RepositoryInfo{FileRoot{ std::filesystem::path{"test/buildtool/execution_engine/executor"}}}; repo_config->Reset(); repo_config->SetInfo("", std::move(info)); } static inline void RunBlobUpload(RepositoryConfig* repo_config, ApiFactory const& factory) { SetupConfig(repo_config); auto api = factory(); HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto const blob = ArtifactBlob::FromMemory(hash_function, ObjectType::File, "test"); REQUIRE(blob.has_value()); CHECK(api->Upload({*blob})); } [[nodiscard]] static inline auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/execution_engine/executor"; } template [[nodiscard]] static inline auto AddAndProcessTree(DependencyGraph* g, Executor* runner, Tree const& tree_desc) -> std::optional { REQUIRE(g->AddAction(tree_desc.Action())); // obtain tree action and tree artifact auto const* tree_action = g->ActionNodeWithId(tree_desc.Id()); REQUIRE_FALSE(tree_action == nullptr); auto const* tree_artifact = g->ArtifactNodeWithId(tree_desc.Output().Id()); REQUIRE_FALSE(tree_artifact == nullptr); // "run" tree action to produce tree artifact REQUIRE(runner->Process(tree_action)); // read computed tree artifact info (digest + object type) return tree_artifact->Content().Info(); } static inline void RunHelloWorldCompilation( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, bool is_hermetic = true, int expected_queued = 0, int expected_cached = 0) { using path = std::filesystem::path; SetupConfig(repo_config); auto const main_cpp_desc = ArtifactDescription::CreateLocal(path{"data/hello_world/main.cpp"}, ""); auto const& main_cpp_id = main_cpp_desc.Id(); std::string const make_hello_id = "make_hello"; auto* env_path = std::getenv("PATH"); std::map env{}; if (env_path != nullptr) { env.emplace("PATH", env_path); } else { env.emplace("PATH", "/bin:/usr/bin"); } auto const make_hello_desc = ActionDescription{ {"out/hello_world"}, {}, Action{make_hello_id, {"c++", "src/main.cpp", "-o", "out/hello_world"}, env}, {{"src/main.cpp", main_cpp_desc}}}; auto const exec_desc = ArtifactDescription::CreateAction(make_hello_id, "out/hello_world"); auto const& exec_id = exec_desc.Id(); DependencyGraph g; CHECK(g.AddAction(make_hello_desc)); CHECK(g.ArtifactNodeWithId(exec_id)->HasBuilderAction()); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); RetryConfig retry_config{}; // default retry config RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = &*remote_config}; auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; // upload local artifacts auto const* main_cpp_node = g.ArtifactNodeWithId(main_cpp_id); CHECK(main_cpp_node != nullptr); CHECK(runner.Process(main_cpp_node)); // process action CHECK(runner.Process(g.ArtifactNodeWithId(exec_id)->BuilderActionNode())); if (is_hermetic) { CHECK(stats->ActionsQueuedCounter() == expected_queued); CHECK(stats->ActionsCachedCounter() == expected_cached); } auto tmpdir = GetTestDir(); // retrieve ALL artifacts REQUIRE(FileSystemManager::CreateDirectory(tmpdir)); for (auto const& artifact_id : g.ArtifactIdentifiers()) { REQUIRE(g.ArtifactNodeWithId(artifact_id)->Content().Info()); CHECK(api->RetrieveToPaths( {*g.ArtifactNodeWithId(artifact_id)->Content().Info()}, {(tmpdir / "output").string()})); CHECK(FileSystemManager::IsFile(tmpdir / "output")); REQUIRE(FileSystemManager::RemoveFile(tmpdir / "output")); } } static inline void RunGreeterCompilation( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, std::string const& greetcpp, bool is_hermetic = true, int expected_queued = 0, int expected_cached = 0) { using path = std::filesystem::path; SetupConfig(repo_config); auto const greet_hpp_desc = ArtifactDescription::CreateLocal(path{"data/greeter/greet.hpp"}, ""); auto const& greet_hpp_id = greet_hpp_desc.Id(); auto const greet_cpp_desc = ArtifactDescription::CreateLocal(path{"data/greeter"} / greetcpp, ""); auto const& greet_cpp_id = greet_cpp_desc.Id(); std::string const compile_greet_id = "compile_greet"; auto* env_path = std::getenv("PATH"); std::map env{}; if (env_path != nullptr) { env.emplace("PATH", env_path); } else { env.emplace("PATH", "/bin:/usr/bin"); } auto const compile_greet_desc = ActionDescription{{"out/greet.o"}, {}, Action{compile_greet_id, {"c++", "-c", "src/greet.cpp", "-I", "include", "-o", "out/greet.o"}, env}, {{"include/greet.hpp", greet_hpp_desc}, {"src/greet.cpp", greet_cpp_desc}}}; auto const greet_o_desc = ArtifactDescription::CreateAction(compile_greet_id, "out/greet.o"); auto const& greet_o_id = greet_o_desc.Id(); std::string const make_lib_id = "make_lib"; auto const make_lib_desc = ActionDescription{ {"out/libgreet.a"}, {}, Action{make_lib_id, {"ar", "rcs", "out/libgreet.a", "greet.o"}, env}, {{"greet.o", greet_o_desc}}}; auto const main_cpp_desc = ArtifactDescription::CreateLocal(path{"data/greeter/main.cpp"}, ""); auto const& main_cpp_id = main_cpp_desc.Id(); auto const libgreet_desc = ArtifactDescription::CreateAction(make_lib_id, "out/libgreet.a"); auto const& libgreet_id = libgreet_desc.Id(); std::string const make_exe_id = "make_exe"; auto const make_exe_desc = ActionDescription{{"out/greeter"}, {}, Action{make_exe_id, {"c++", "src/main.cpp", "-I", "include", "-L", "lib", "-lgreet", "-o", "out/greeter"}, env}, {{"src/main.cpp", main_cpp_desc}, {"include/greet.hpp", greet_hpp_desc}, {"lib/libgreet.a", libgreet_desc}}}; auto const exec_id = ArtifactDescription::CreateAction(make_exe_id, "out/greeter").Id(); DependencyGraph g; CHECK(g.Add({compile_greet_desc, make_lib_desc, make_exe_desc})); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); RetryConfig retry_config{}; // default retry config RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = &*remote_config}; auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; // upload local artifacts for (auto const& id : {greet_hpp_id, greet_cpp_id, main_cpp_id}) { auto const* node = g.ArtifactNodeWithId(id); CHECK(node != nullptr); CHECK(runner.Process(node)); } // process actions CHECK( runner.Process(g.ArtifactNodeWithId(greet_o_id)->BuilderActionNode())); CHECK( runner.Process(g.ArtifactNodeWithId(libgreet_id)->BuilderActionNode())); CHECK(runner.Process(g.ArtifactNodeWithId(exec_id)->BuilderActionNode())); if (is_hermetic) { CHECK(stats->ActionsQueuedCounter() == expected_queued); CHECK(stats->ActionsCachedCounter() == expected_cached); } auto tmpdir = GetTestDir(); // retrieve ALL artifacts REQUIRE(FileSystemManager::CreateDirectory(tmpdir)); for (auto const& artifact_id : g.ArtifactIdentifiers()) { REQUIRE(g.ArtifactNodeWithId(artifact_id)->Content().Info()); CHECK(api->RetrieveToPaths( {*g.ArtifactNodeWithId(artifact_id)->Content().Info()}, {(tmpdir / "output").string()})); CHECK(FileSystemManager::IsFile(tmpdir / "output")); REQUIRE(FileSystemManager::RemoveFile(tmpdir / "output")); } } [[maybe_unused]] static void TestBlobUpload(RepositoryConfig* repo_config, ApiFactory const& factory) { SetupConfig(repo_config); // NOLINTNEXTLINE RunBlobUpload(repo_config, factory); } [[maybe_unused]] static void TestHelloWorldCompilation( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, bool is_hermetic = true) { SetupConfig(repo_config); // expecting 1 action queued, 0 results from cache // NOLINTNEXTLINE RunHelloWorldCompilation( repo_config, stats, progress, factory, auth, is_hermetic, 1, 0); SECTION("Running same compilation again") { // expecting 2 actions queued, 1 result from cache // NOLINTNEXTLINE RunHelloWorldCompilation( repo_config, stats, progress, factory, auth, is_hermetic, 2, 1); } } [[maybe_unused]] static void TestGreeterCompilation( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, bool is_hermetic = true) { SetupConfig(repo_config); // expecting 3 action queued, 0 results from cache // NOLINTNEXTLINE RunGreeterCompilation(repo_config, stats, progress, factory, auth, "greet.cpp", is_hermetic, 3, 0); SECTION("Running same compilation again") { // expecting 6 actions queued, 3 results from cache RunGreeterCompilation(repo_config, stats, progress, factory, auth, "greet.cpp", is_hermetic, 6, // NOLINT 3); } SECTION("Running modified compilation") { // expecting 6 actions queued, 2 results from cache RunGreeterCompilation(repo_config, stats, progress, factory, auth, "greet_mod.cpp", is_hermetic, 6, // NOLINT 2); } } static inline void TestUploadAndDownloadTrees( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, bool /*is_hermetic*/ = true, int /*expected_queued*/ = 0, int /*expected_cached*/ = 0) { SetupConfig(repo_config); auto tmpdir = GetTestDir(); auto* env_path = std::getenv("PATH"); std::map env{}; if (env_path != nullptr) { env.emplace("PATH", env_path); } else { env.emplace("PATH", "/bin:/usr/bin"); } HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; auto const foo = ArtifactBlob::FromMemory(hash_function, ObjectType::File, "foo"); REQUIRE(foo.has_value()); auto const bar = ArtifactBlob::FromMemory(hash_function, ObjectType::File, "bar"); REQUIRE(bar.has_value()); // upload blobs auto api = factory(); REQUIRE(api->Upload({*foo, *bar})); // define known artifacts auto foo_desc = ArtifactDescription::CreateKnown(foo->GetDigest(), ObjectType::File); auto bar_desc = ArtifactDescription::CreateKnown(bar->GetDigest(), ObjectType::Symlink); DependencyGraph g{}; auto foo_id = g.AddArtifact(foo_desc); auto bar_id = g.AddArtifact(bar_desc); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); RetryConfig retry_config{}; // default retry config RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = &*remote_config}; auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; REQUIRE(runner.Process(g.ArtifactNodeWithId(foo_id))); REQUIRE(runner.Process(g.ArtifactNodeWithId(bar_id))); SECTION("Simple tree") { auto tree_desc = Tree{{{"a", foo_desc}, {"b", bar_desc}}}; auto tree_info = AddAndProcessTree(&g, &runner, tree_desc); REQUIRE(tree_info); CHECK(IsTreeObject(tree_info->type)); tmpdir /= "simple"; CHECK(api->RetrieveToPaths({*tree_info}, {tmpdir.string()})); CHECK(FileSystemManager::IsDirectory(tmpdir)); CHECK(FileSystemManager::IsFile(tmpdir / "a")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "b")); CHECK(*FileSystemManager::ReadFile(tmpdir / "a") == "foo"); CHECK(*FileSystemManager::ReadSymlink(tmpdir / "b") == "bar"); REQUIRE(FileSystemManager::RemoveDirectory(tmpdir)); } SECTION("Subdir in tree path") { auto tree_desc = Tree{{{"a", foo_desc}, {"b/a", bar_desc}}}; auto tree_info = AddAndProcessTree(&g, &runner, tree_desc); REQUIRE(tree_info); CHECK(IsTreeObject(tree_info->type)); tmpdir /= "subdir"; CHECK(api->RetrieveToPaths({*tree_info}, {tmpdir.string()})); CHECK(FileSystemManager::IsDirectory(tmpdir)); CHECK(FileSystemManager::IsDirectory(tmpdir / "b")); CHECK(FileSystemManager::IsFile(tmpdir / "a")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "b" / "a")); CHECK(*FileSystemManager::ReadFile(tmpdir / "a") == "foo"); CHECK(*FileSystemManager::ReadSymlink(tmpdir / "b" / "a") == "bar"); REQUIRE(FileSystemManager::RemoveDirectory(tmpdir)); } SECTION("Nested trees") { auto tree_desc_nested = Tree{{{"a", bar_desc}}}; auto tree_desc_parent = Tree{{{"a", foo_desc}, {"b", tree_desc_nested.Output()}}}; REQUIRE(AddAndProcessTree(&g, &runner, tree_desc_nested)); auto tree_info = AddAndProcessTree(&g, &runner, tree_desc_parent); REQUIRE(tree_info); CHECK(IsTreeObject(tree_info->type)); tmpdir /= "nested"; CHECK(api->RetrieveToPaths({*tree_info}, {tmpdir.string()})); CHECK(FileSystemManager::IsDirectory(tmpdir)); CHECK(FileSystemManager::IsDirectory(tmpdir / "b")); CHECK(FileSystemManager::IsFile(tmpdir / "a")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "b" / "a")); CHECK(*FileSystemManager::ReadFile(tmpdir / "a") == "foo"); CHECK(*FileSystemManager::ReadSymlink(tmpdir / "b" / "a") == "bar"); REQUIRE(FileSystemManager::RemoveDirectory(tmpdir)); } SECTION("Dot-path tree as action input") { auto tree_desc = Tree{{{"a", foo_desc}, {"b/a", bar_desc}}}; auto action_inputs = ActionDescription::inputs_t{{".", tree_desc.Output()}}; ActionDescription action_desc{{"a", "b/a"}, {}, Action{"action_id", {"echo"}, env}, action_inputs}; REQUIRE(AddAndProcessTree(&g, &runner, tree_desc)); REQUIRE(g.Add({action_desc})); auto const* action_node = g.ActionNodeWithId("action_id"); REQUIRE(runner.Process(action_node)); tmpdir /= "dotpath"; std::vector infos{}; std::vector paths{}; for (auto const& [path, node] : action_node->OutputFiles()) { REQUIRE(node->Content().Info()); paths.emplace_back(tmpdir / path); infos.emplace_back(*node->Content().Info()); } CHECK(api->RetrieveToPaths(infos, paths)); CHECK(FileSystemManager::IsDirectory(tmpdir)); CHECK(FileSystemManager::IsDirectory(tmpdir / "b")); CHECK(FileSystemManager::IsFile(tmpdir / "a")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "b" / "a")); CHECK(*FileSystemManager::ReadFile(tmpdir / "a") == "foo"); CHECK(*FileSystemManager::ReadSymlink(tmpdir / "b" / "a") == "bar"); REQUIRE(FileSystemManager::RemoveDirectory(tmpdir)); } SECTION("Dot-path non-tree as action input") { auto action_inputs = ActionDescription::inputs_t{{".", foo_desc}}; ActionDescription action_desc{ {"foo"}, {}, Action{"action_id", std::vector{"echo"}, {}}, action_inputs}; REQUIRE(g.Add({action_desc})); auto const* action_node = g.ActionNodeWithId("action_id"); REQUIRE_FALSE(runner.Process(action_node)); } } static inline void TestRetrieveOutputDirectories( RepositoryConfig* repo_config, gsl::not_null const& stats, gsl::not_null const& progress, ApiFactory const& factory, gsl::not_null const& auth, bool /*is_hermetic*/ = true, int /*expected_queued*/ = 0, int /*expected_cached*/ = 0) { SetupConfig(repo_config); auto const make_tree_id = std::string{"make_tree"}; auto const* make_tree_cmd = "mkdir -p baz/baz/\n" "touch foo\n" "ln -s dummy bar\n" "touch baz/foo\n" "ln -s dummy baz/bar\n" "touch baz/baz/foo\n" "ln -s dummy baz/baz/bar"; auto create_action = [&make_tree_id, make_tree_cmd]( std::vector&& out_files, std::vector&& out_dirs) { auto* env_path = std::getenv("PATH"); std::map env{}; if (env_path != nullptr) { env.emplace("PATH", env_path); } else { env.emplace("PATH", "/bin:/usr/bin"); } return ActionDescription{ std::move(out_files), std::move(out_dirs), Action{make_tree_id, {"sh", "-c", make_tree_cmd}, env}, {}}; }; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); RetryConfig retry_config{}; // default retry config RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = &*remote_config}; SECTION("entire action output as directory") { auto const make_tree_desc = create_action({}, {""}); auto const root_desc = ArtifactDescription::CreateAction(make_tree_id, ""); DependencyGraph g{}; REQUIRE(g.AddAction(make_tree_desc)); auto const* action = g.ActionNodeWithId(make_tree_id); REQUIRE_FALSE(action == nullptr); auto const* root = g.ArtifactNodeWithId(root_desc.Id()); REQUIRE_FALSE(root == nullptr); // run action auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; REQUIRE(runner.Process(action)); // read output auto root_info = root->Content().Info(); REQUIRE(root_info); CHECK(IsTreeObject(root_info->type)); // retrieve ALL artifacts auto tmpdir = GetTestDir() / "entire_output"; REQUIRE(FileSystemManager::CreateDirectory(tmpdir)); REQUIRE(api->RetrieveToPaths({*root_info}, {tmpdir})); CHECK(FileSystemManager::IsFile(tmpdir / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "bar")); CHECK(FileSystemManager::IsDirectory(tmpdir / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "baz" / "bar")); CHECK(FileSystemManager::IsDirectory(tmpdir / "baz" / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "baz" / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "baz" / "baz" / "bar")); } SECTION("disjoint files and directories") { auto const make_tree_desc = create_action({"foo", "bar"}, {"baz"}); auto const foo_desc = ArtifactDescription::CreateAction(make_tree_id, "foo"); auto const bar_desc = ArtifactDescription::CreateAction(make_tree_id, "bar"); auto const baz_desc = ArtifactDescription::CreateAction(make_tree_id, "baz"); DependencyGraph g{}; REQUIRE(g.AddAction(make_tree_desc)); auto const* action = g.ActionNodeWithId(make_tree_id); REQUIRE_FALSE(action == nullptr); auto const* foo = g.ArtifactNodeWithId(foo_desc.Id()); REQUIRE_FALSE(foo == nullptr); auto const* bar = g.ArtifactNodeWithId(bar_desc.Id()); REQUIRE_FALSE(bar == nullptr); auto const* baz = g.ArtifactNodeWithId(baz_desc.Id()); REQUIRE_FALSE(baz == nullptr); // run action auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; REQUIRE(runner.Process(action)); // read output auto foo_info = foo->Content().Info(); REQUIRE(foo_info); CHECK(IsFileObject(foo_info->type)); auto bar_info = bar->Content().Info(); REQUIRE(bar_info); CHECK(IsSymlinkObject(bar_info->type)); auto baz_info = baz->Content().Info(); REQUIRE(baz_info); CHECK(IsTreeObject(baz_info->type)); // retrieve ALL artifacts auto tmpdir = GetTestDir() / "disjoint"; REQUIRE(FileSystemManager::CreateDirectory(tmpdir)); REQUIRE(api->RetrieveToPaths({*foo_info}, {tmpdir / "foo"})); CHECK(FileSystemManager::IsFile(tmpdir / "foo")); REQUIRE(api->RetrieveToPaths({*bar_info}, {tmpdir / "bar"})); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "bar")); REQUIRE(api->RetrieveToPaths({*baz_info}, {tmpdir / "baz"})); CHECK(FileSystemManager::IsDirectory(tmpdir / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "baz" / "bar")); CHECK(FileSystemManager::IsDirectory(tmpdir / "baz" / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "baz" / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "baz" / "baz" / "bar")); } SECTION("nested files and directories") { auto const make_tree_desc = create_action({"foo", "baz/bar"}, {"", "baz/baz"}); auto const root_desc = ArtifactDescription::CreateAction(make_tree_id, ""); auto const foo_desc = ArtifactDescription::CreateAction(make_tree_id, "foo"); auto const bar_desc = ArtifactDescription::CreateAction(make_tree_id, "baz/bar"); auto const baz_desc = ArtifactDescription::CreateAction(make_tree_id, "baz/baz"); DependencyGraph g{}; REQUIRE(g.AddAction(make_tree_desc)); auto const* action = g.ActionNodeWithId(make_tree_id); REQUIRE_FALSE(action == nullptr); auto const* root = g.ArtifactNodeWithId(root_desc.Id()); REQUIRE_FALSE(root == nullptr); auto const* foo = g.ArtifactNodeWithId(foo_desc.Id()); REQUIRE_FALSE(foo == nullptr); auto const* bar = g.ArtifactNodeWithId(bar_desc.Id()); REQUIRE_FALSE(bar == nullptr); auto const* baz = g.ArtifactNodeWithId(baz_desc.Id()); REQUIRE_FALSE(baz == nullptr); // run action auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{.repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; REQUIRE(runner.Process(action)); // read output auto root_info = root->Content().Info(); REQUIRE(root_info); CHECK(IsTreeObject(root_info->type)); auto foo_info = foo->Content().Info(); REQUIRE(foo_info); CHECK(IsFileObject(foo_info->type)); auto bar_info = bar->Content().Info(); REQUIRE(bar_info); CHECK(IsSymlinkObject(bar_info->type)); auto baz_info = baz->Content().Info(); REQUIRE(baz_info); CHECK(IsTreeObject(baz_info->type)); // retrieve ALL artifacts auto tmpdir = GetTestDir() / "baz"; REQUIRE(FileSystemManager::CreateDirectory(tmpdir)); REQUIRE(api->RetrieveToPaths({*root_info}, {tmpdir / "root"})); CHECK(FileSystemManager::IsDirectory(tmpdir / "root")); CHECK(FileSystemManager::IsFile(tmpdir / "root" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "root" / "bar")); CHECK(FileSystemManager::IsDirectory(tmpdir / "root" / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "root" / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "root" / "baz" / "bar")); CHECK(FileSystemManager::IsDirectory(tmpdir / "root" / "baz" / "baz")); CHECK( FileSystemManager::IsFile(tmpdir / "root" / "baz" / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "root" / "baz" / "baz" / "bar")); REQUIRE(api->RetrieveToPaths({*foo_info}, {tmpdir / "foo"})); CHECK(FileSystemManager::IsFile(tmpdir / "foo")); REQUIRE(api->RetrieveToPaths({*bar_info}, {tmpdir / "bar"})); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "bar")); REQUIRE(api->RetrieveToPaths({*baz_info}, {tmpdir / "baz"})); CHECK(FileSystemManager::IsDirectory(tmpdir / "baz")); CHECK(FileSystemManager::IsFile(tmpdir / "baz" / "foo")); CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "baz" / "bar")); } SECTION("non-existing outputs") { SECTION("non-existing file") { auto const make_tree_desc = create_action({"fool"}, {}); auto const fool_desc = ArtifactDescription::CreateAction(make_tree_id, "fool"); DependencyGraph g{}; REQUIRE(g.AddAction(make_tree_desc)); auto const* action = g.ActionNodeWithId(make_tree_id); REQUIRE_FALSE(action == nullptr); auto const* fool = g.ArtifactNodeWithId(fool_desc.Id()); REQUIRE_FALSE(fool == nullptr); // run action auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{ .repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK_FALSE(runner.Process(action)); } SECTION("non-existing directory") { auto const make_tree_desc = create_action({"bazel"}, {}); auto const bazel_desc = ArtifactDescription::CreateAction(make_tree_id, "bazel"); DependencyGraph g{}; REQUIRE(g.AddAction(make_tree_desc)); auto const* action = g.ActionNodeWithId(make_tree_id); REQUIRE_FALSE(action == nullptr); auto const* bazel = g.ArtifactNodeWithId(bazel_desc.Id()); REQUIRE_FALSE(bazel == nullptr); // run action auto api = factory(); auto const apis = CreateTestApiBundle(api); ExecutionContext const exec_context{ .repo_config = repo_config, .apis = &apis, .remote_context = &remote_context, .statistics = stats, .progress = progress, .profile = std::nullopt}; Executor runner{&exec_context}; CHECK_FALSE(runner.Process(action)); } } } #endif // INCLUDED_SRC_TEST_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_API_TEST_HPP executor_api_local.test.cpp000066400000000000000000000127321516554100600352320ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/buildtool/execution_engine/executor/executor_api.test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" TEST_CASE("Executor: Upload blob", "[executor]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; TestBlobUpload(&repo_config, [&] { return std::make_unique(&local_context, &repo_config); }); } TEST_CASE("Executor: Compile hello world", "[executor]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestHelloWorldCompilation( &repo_config, &stats, &progress, [&] { return std::make_unique(&local_context, &repo_config); }, &*auth_config); } TEST_CASE("Executor: Compile greeter", "[executor]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestGreeterCompilation( &repo_config, &stats, &progress, [&] { return std::make_unique(&local_context, &repo_config); }, &*auth_config); } TEST_CASE("Executor: Upload and download trees", "[executor]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestUploadAndDownloadTrees( &repo_config, &stats, &progress, [&] { return std::make_unique(&local_context, &repo_config); }, &*auth_config); } TEST_CASE("Executor: Retrieve output directories", "[executor]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); LocalExecutionConfig local_exec_config{}; // pack the local context instances to be passed to LocalApi LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestRetrieveOutputDirectories( &repo_config, &stats, &progress, [&] { return std::make_unique(&local_context, &repo_config); }, &*auth_config); } executor_api_remote_bazel.test.cpp000066400000000000000000000155021516554100600366060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/executor// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/storage/config.hpp" #include "test/buildtool/execution_engine/executor/executor_api.test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" TEST_CASE("Executor: Upload blob", "[executor]") { RepositoryConfig repo_config{}; ExecutionConfiguration config; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); RetryConfig retry_config{}; // default retry config auto storage_config = TestStorageConfig::Create(); TestBlobUpload(&repo_config, [&] { return std::make_shared( "remote-execution", remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, config, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")); }); } TEST_CASE("Executor: Compile hello world", "[executor]") { RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; ExecutionConfiguration config; config.skip_cache_lookup = false; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); RetryConfig retry_config{}; // default retry config auto storage_config = TestStorageConfig::Create(); TestHelloWorldCompilation( &repo_config, &stats, &progress, [&] { return std::make_shared( "remote-execution", remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, config, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); } TEST_CASE("Executor: Compile greeter", "[executor]") { RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; ExecutionConfiguration config; config.skip_cache_lookup = false; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); RetryConfig retry_config{}; // default retry config auto storage_config = TestStorageConfig::Create(); TestGreeterCompilation( &repo_config, &stats, &progress, [&] { return std::make_shared( "remote-execution", remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, config, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); } TEST_CASE("Executor: Upload and download trees", "[executor]") { RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; ExecutionConfiguration config; config.skip_cache_lookup = false; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); RetryConfig retry_config{}; // default retry config auto storage_config = TestStorageConfig::Create(); TestUploadAndDownloadTrees( &repo_config, &stats, &progress, [&] { return std::make_shared( "remote-execution", remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, config, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); } TEST_CASE("Executor: Retrieve output directories", "[executor]") { RepositoryConfig repo_config{}; Statistics stats{}; Progress progress{}; ExecutionConfiguration config; config.skip_cache_lookup = false; auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); REQUIRE(remote_config->remote_address); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); RetryConfig retry_config{}; // default retry config auto storage_config = TestStorageConfig::Create(); TestRetrieveOutputDirectories( &repo_config, &stats, &progress, [&] { return std::make_shared( "remote-execution", remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, config, storage_config.Get().hash_function, storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/traverser/000077500000000000000000000000001516554100600301405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/traverser/TARGETS000066400000000000000000000015371516554100600312020ustar00rootroot00000000000000{ "traverser": { "type": ["@", "rules", "CC/test", "test"] , "name": ["traverser"] , "srcs": ["traverser.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/execution_engine/traverser", "traverser"] , ["", "catch-main"] , ["utils", "container_matchers"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "execution_engine", "traverser"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["traverser"] , "deps": ["traverser"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/traverser/traverser.test.cpp000066400000000000000000001037731516554100600336520ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/execution_engine/traverser/traverser.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/identifier.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "test/utils/container_matchers.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" namespace { auto const kNumJobs = std::max(1U, std::thread::hardware_concurrency()); [[nodiscard]] auto IsArtifactLocal(nlohmann::json const& j) -> bool { return j.at("type") == "LOCAL"; } class TestBuildInfo { public: [[nodiscard]] auto CorrectlyBuilt() const noexcept -> std::unordered_set { return correctly_built_; } [[nodiscard]] auto IncorrectlyBuilt() const noexcept -> std::unordered_set { return incorrectly_built_; } [[nodiscard]] auto ArtifactsUploaded() const noexcept -> std::unordered_set { return artifacts_uploaded_; } [[nodiscard]] auto WasUploadRepeated() noexcept -> bool { return not uploaded_more_than_once_.empty(); } [[nodiscard]] auto Name() const noexcept -> std::string { return name_; } void SetName(std::string const& name) noexcept { std::lock_guard lock{mutex_}; name_ = name; } void SetName(std::string&& name) noexcept { std::lock_guard lock{mutex_}; name_ = std::move(name); } [[nodiscard]] auto InsertCorrectlyBuilt( ArtifactIdentifier const& artifact_id) -> bool { std::lock_guard lock{mutex_}; auto const [_, first_time_added] = correctly_built_.insert(artifact_id); return first_time_added; } [[nodiscard]] auto InsertIncorrectlyBuilt( ArtifactIdentifier const& artifact_id) -> bool { std::lock_guard lock{mutex_}; auto const [_, first_time_added] = incorrectly_built_.insert(artifact_id); return first_time_added; } auto InsertArtifactUploaded(ArtifactIdentifier const& artifact_id) -> bool { std::lock_guard lock{mutex_}; auto const [_, first_time_added] = artifacts_uploaded_.insert(artifact_id); if (not first_time_added) { uploaded_more_than_once_.insert(artifact_id); } return true; } private: std::unordered_set correctly_built_; std::unordered_set incorrectly_built_; std::unordered_set artifacts_uploaded_; std::unordered_set uploaded_more_than_once_; std::string name_; std::mutex mutex_; }; class TestExecutor { public: explicit TestExecutor(TestBuildInfo* info) noexcept : name_{info->Name()}, build_info_{info} {} [[nodiscard]] auto Process( gsl::not_null const& action) const noexcept -> bool { try { build_info_->SetName(name_); bool const all_deps_available = AllAvailable(action->Children()); if (all_deps_available) { return std::all_of( action->OutputFiles().begin(), action->OutputFiles().end(), [this](auto const& entry) { auto const& [name, node] = entry; if (not build_info_->InsertCorrectlyBuilt( node->Content().Id())) { [[maybe_unused]] auto was_it_added = build_info_->InsertIncorrectlyBuilt( node->Content().Id()); return false; } return true; }); } for (auto const& [name, node] : action->OutputFiles()) { [[maybe_unused]] auto was_it_added = build_info_->InsertIncorrectlyBuilt(node->Content().Id()); } return false; } catch (...) { return false; } } [[nodiscard]] auto Process( gsl::not_null const& artifact) const noexcept -> bool { try { build_info_->InsertArtifactUploaded(artifact->Content().Id()); } catch (...) { return false; } return true; } private: std::string const name_; TestBuildInfo* build_info_; template // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) [[nodiscard]] auto AllAvailable(Container&& c) const noexcept -> bool { return std::all_of(std::begin(c), std::end(c), [](auto node) { return node->TraversalState()->IsAvailable(); }); } }; // Class to simplify the writing of tests, checking that no outputs are repeated // and keeping track of what needs to be built class TestProject { public: auto AddOutputInputPair(std::string const& action_id, std::vector const& outputs, std::vector const& inputs) -> bool { std::vector command; command.emplace_back("BUILD"); for (auto const& output : outputs) { command.push_back(output); auto const out_id = ArtifactDescription::CreateAction( action_id, std::filesystem::path{output}) .Id(); auto [_, is_inserted] = artifacts_to_be_built_.insert(out_id); if (not is_inserted) { return false; } } auto inputs_desc = ActionDescription::inputs_t{}; if (not inputs.empty()) { command.emplace_back("FROM"); auto const hash_type = TestHashType::ReadFromEnvironment(); for (auto const& input_desc : inputs) { auto artifact = ArtifactDescription::FromJson(hash_type, input_desc); REQUIRE(artifact); auto const input_id = artifact->Id(); command.push_back(input_id); inputs_desc.emplace(input_id, *artifact); if (IsArtifactLocal(input_desc)) { local_artifacts_.insert(input_id); } } } graph_full_description_.emplace_back(ActionDescription{ outputs, {}, Action{action_id, command, {}}, inputs_desc}); return true; } auto FillGraph(gsl::not_null const& g) -> bool { return g->Add(graph_full_description_); } [[nodiscard]] auto ArtifactsToBeBuilt() const noexcept -> std::unordered_set { return artifacts_to_be_built_; } [[nodiscard]] auto LocalArtifacts() const noexcept -> std::unordered_set { return local_artifacts_; } private: std::vector graph_full_description_; std::unordered_set artifacts_to_be_built_; std::unordered_set local_artifacts_; }; } // namespace TEST_CASE("Executable", "[traverser]") { TestProject p; CHECK(p.AddOutputInputPair( "action", {"executable"}, {ArtifactDescription::CreateLocal("main.cpp", "").ToJson()})); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Traverse()") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(executable)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const exec_id = ArtifactDescription::CreateAction("action", "executable").Id(); auto const traversed = traverser.Traverse({exec_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } TEST_CASE("Executable depends on library", "[traverser]") { TestProject p; CHECK(p.AddOutputInputPair( "make_exe", {"executable"}, {ArtifactDescription::CreateLocal("main.cpp", "repo").ToJson(), ArtifactDescription::CreateAction("make_lib", "library").ToJson()})); CHECK(p.AddOutputInputPair( "make_lib", {"library"}, {ArtifactDescription::CreateLocal("library.hpp", "repo").ToJson(), ArtifactDescription::CreateLocal("library.cpp", "repo").ToJson()})); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Full build (without specifying artifacts)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Full build (executable)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const exec_id = ArtifactDescription::CreateAction("make_exe", "executable") .Id(); CHECK(traverser.Traverse({exec_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Only build library") { auto const lib_id = ArtifactDescription::CreateAction("make_lib", "library").Id(); { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({lib_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( {lib_id})); CHECK(build_info.IncorrectlyBuilt().empty()); auto const lib_cpp_id = ArtifactDescription::CreateLocal("library.cpp", "repo").Id(); auto const lib_hpp_id = ArtifactDescription::CreateLocal("library.hpp", "repo").Id(); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( {lib_cpp_id, lib_hpp_id})); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } TEST_CASE("Two artifacts depend on another", "[traverser]") { TestProject p; auto const description = ArtifactDescription::CreateAction("make_dep", "dep"); auto const dep_desc = description.ToJson(); auto const& dep_id = description.Id(); CHECK(p.AddOutputInputPair("action1", {"toplevel1"}, {dep_desc})); CHECK(p.AddOutputInputPair("action2", {"toplevel2"}, {dep_desc})); CHECK(p.AddOutputInputPair( "make_dep", {"dep"}, {ArtifactDescription::CreateLocal("leaf1", "repo").ToJson(), ArtifactDescription::CreateLocal("leaf2", "repo").ToJson()})); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Full build") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Only specified top-level artifact is built") { auto const toplevel1_id = ArtifactDescription::CreateAction("action1", "toplevel1").Id(); { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({toplevel1_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( {toplevel1_id, dep_id})); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK(build_info.Name() == name); } } TEST_CASE("Action with two outputs, no deps", "[traverser]") { TestProject p; CHECK(p.AddOutputInputPair("make_outputs", {"output1", "output2"}, {})); auto const output1_id = ArtifactDescription::CreateAction("make_outputs", "output1").Id(); auto const output2_id = ArtifactDescription::CreateAction("make_outputs", "output2").Id(); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Traverse()") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(output1)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({output1_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(output1, output2)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({output1_id, output2_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } TEST_CASE("Action with two outputs, one dep", "[traverser]") { TestProject p; CHECK(p.AddOutputInputPair( "make_outputs", {"output1", "output2"}, {ArtifactDescription::CreateLocal("dep", "repo").ToJson()})); auto const output1_id = ArtifactDescription::CreateAction("make_outputs", "output1").Id(); auto const output2_id = ArtifactDescription::CreateAction("make_outputs", "output2").Id(); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Traverse()") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(output1)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({output1_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(output1, output2)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({output1_id, output2_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(dep, output2)") { auto const dep_id = ArtifactDescription::CreateLocal("dep", "repo").Id(); { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({dep_id, output2_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } TEST_CASE("Action with two outputs, actions depend on each of outputs", "[traverser]") { TestProject p; CHECK(p.AddOutputInputPair("make_outputs", {"output1", "output2"}, {})); auto const desc_1 = ArtifactDescription::CreateAction("make_outputs", "output1"); auto const output1_desc = desc_1.ToJson(); auto const& output1_id = desc_1.Id(); auto const desc_2 = ArtifactDescription::CreateAction("make_outputs", "output2"); auto const output2_desc = desc_2.ToJson(); auto const& output2_id = desc_2.Id(); CHECK(p.AddOutputInputPair("consumer1", {"exec1"}, {output1_desc})); auto const exec1_id = ArtifactDescription::CreateAction("consumer1", "exec1").Id(); CHECK(p.AddOutputInputPair("consumer2", {"exec2"}, {output2_desc})); auto const exec2_id = ArtifactDescription::CreateAction("consumer2", "exec2").Id(); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted"; build_info.SetName(name); SECTION("Traverse()") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(exec1)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({exec1_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( {exec1_id, output1_id, output2_id})); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(exec2, output1)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({output1_id, exec2_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( {exec2_id, output1_id, output2_id})); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Traverse(exec1, exec2)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); auto const traversed = traverser.Traverse({exec1_id, exec2_id}); CHECK(traversed); } CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } TEST_CASE("lib2 depends on lib1, executable depends on lib1 and lib2") { TestProject p; auto const desc_1 = ArtifactDescription::CreateAction("make_lib1", "lib1"); auto const lib1_desc = desc_1.ToJson(); auto const& lib1_id = desc_1.Id(); auto const desc_2 = ArtifactDescription::CreateAction("make_lib2", "lib2"); auto const lib2_desc = desc_2.ToJson(); auto const& lib2_id = desc_2.Id(); auto const exec_id = ArtifactDescription::CreateAction("make_exe", "executable").Id(); CHECK(p.AddOutputInputPair( "make_exe", {"executable"}, {ArtifactDescription::CreateLocal("main.cpp", "repo").ToJson(), lib1_desc, lib2_desc})); CHECK(p.AddOutputInputPair( "make_lib1", {"lib1"}, {ArtifactDescription::CreateLocal("lib1.hpp", "repo").ToJson(), ArtifactDescription::CreateLocal("lib1.cpp", "repo").ToJson()})); CHECK(p.AddOutputInputPair( "make_lib2", {"lib2"}, {lib1_desc, ArtifactDescription::CreateLocal("lib2.hpp", "repo").ToJson(), ArtifactDescription::CreateLocal("lib2.cpp", "repo").ToJson()})); DependencyGraph g; CHECK(p.FillGraph(&g)); TestBuildInfo build_info; std::atomic failed{}; std::string name = "This is a long name that shouldn't be corrupted "; build_info.SetName(name); SECTION(" Full build(without specifying artifacts) ") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse()); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Full build (executable)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({exec_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Full build (executable + lib1)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({exec_id, lib1_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Full build (executable + lib2)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({exec_id, lib2_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("Full build (executable + lib1 + lib2)") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({exec_id, lib1_id, lib2_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION("First call does not build all artifacts") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({lib1_id})); CHECK(traverser.Traverse({exec_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( p.ArtifactsToBeBuilt())); CHECK(build_info.IncorrectlyBuilt().empty()); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( p.LocalArtifacts())); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } SECTION( "Traverse(lib2), executable is not built even if lib1 would notify its " "action") { { TestExecutor runner{&build_info}; Traverser traverser(runner, g, kNumJobs, &failed); CHECK(traverser.Traverse({lib2_id})); } CHECK_FALSE(failed); CHECK_THAT( build_info.CorrectlyBuilt(), HasSameUniqueElementsAs>( {lib1_id, lib2_id})); CHECK(build_info.IncorrectlyBuilt().empty()); auto const lib1_hpp_id = ArtifactDescription::CreateLocal("lib1.hpp", "repo").Id(); auto const lib1_cpp_id = ArtifactDescription::CreateLocal("lib1.cpp", "repo").Id(); auto const lib2_hpp_id = ArtifactDescription::CreateLocal("lib2.hpp", "repo").Id(); auto const lib2_cpp_id = ArtifactDescription::CreateLocal("lib2.cpp", "repo").Id(); CHECK_THAT( build_info.ArtifactsUploaded(), HasSameUniqueElementsAs>( {lib1_hpp_id, lib1_cpp_id, lib2_hpp_id, lib2_cpp_id})); CHECK_FALSE(build_info.WasUploadRepeated()); CHECK(build_info.Name() == name); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/tree_operations/000077500000000000000000000000001516554100600313255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/tree_operations/TARGETS000066400000000000000000000022361516554100600323640ustar00rootroot00000000000000{ "tree_operations_utils": { "type": ["@", "rules", "CC/test", "test"] , "name": ["tree_operations_utils"] , "srcs": ["tree_operations_utils.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/local", "local_api"] , [ "@" , "src" , "src/buildtool/execution_engine/tree_operations" , "tree_operations_utils" ] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["buildtool/execution_api/common", "api_test"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "tree_operations"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["tree_operations"] , "deps": ["tree_operations_utils"] } } tree_operations_utils.test.cpp000066400000000000000000000124321516554100600373540ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/execution_engine/tree_operations// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); you // may not use this file except in compliance with the License. You may // obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or // implied. See the License for the specific language governing // permissions and limitations under the License. #include "src/buildtool/execution_engine/tree_operations/tree_operations_utils.hpp" #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "test/buildtool/execution_api/common/api_test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" using TreeEntries = TreeOperationsUtils::TreeEntries; using TreeEntry = TreeOperationsUtils::TreeEntry; // Creates a chain of nested two-entry trees of n levels with both // entries at each tree level pointing to the next single subtree. The // tree at the last level points the blobs with the given contents. // // tree_1 --t1--> tree_2 --t1--> tree_3 -- ... --> tree_n --b1--> blob_1 // \---t2----^ \---t2----^ \--- ... ----^ \---b2--> blob_2 // \--- ... [[nodiscard]] static auto CreateNestedTree( int levels, IExecutionApi const& api, HashFunction const& hash_function, std::unordered_map const& blobs) noexcept -> std::optional { if (levels > 1) { // Create subtree with number of levels - 1. auto tree_info = CreateNestedTree(levels - 1, api, hash_function, blobs); if (not tree_info) { return std::nullopt; } // Create tree with two entries pointing to the subtree. TreeEntries entries{}; entries["tree1"] = TreeEntry{.info = *tree_info}; entries["tree2"] = TreeEntry{.info = *tree_info}; auto tree = TreeOperationsUtils::WriteTree(api, entries); if (not tree) { return std::nullopt; } return *tree; } // Create blob entries. TreeEntries entries{}; for (auto const& [blob_name, blob_content] : blobs) { auto blob_info = Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs( hash_function, blob_content), .type = ObjectType::File}; entries[blob_name] = TreeEntry{.info = std::move(blob_info)}; } // Create tree containing the blobs. auto tree = TreeOperationsUtils::WriteTree(api, entries); if (not tree) { return std::nullopt; } return *tree; } TEST_CASE("TreeOperationsUtils", "[tree_operations]") { // Create local execution api. auto local_exec_config = CreateLocalExecConfig(); auto storage_config = TestStorageConfig::Create(); auto storage = Storage::Create(&storage_config.Get()); LocalContext local_context{.exec_config = &local_exec_config, .storage_config = &storage_config.Get(), .storage = &storage}; IExecutionApi::Ptr local_api{new LocalApi{&local_context}}; HashFunction hash_function{local_api->GetHashType()}; SECTION("No duplicated tree-overlay calculations") { // Create two long nested trees. constexpr int kTreeLevels = 65; auto base_tree_info = CreateNestedTree( kTreeLevels, *local_api, hash_function, {{"foo", "foo"}}); REQUIRE(base_tree_info); auto other_tree_info = CreateNestedTree( kTreeLevels, *local_api, hash_function, {{"bar", "bar"}}); REQUIRE(other_tree_info); // Compute tree overlay. A naive tree-overlay computation of // these trees has a time complexity of O(2^n). A properly // deduplicated tree-overlay computation has only O(n) and will // finish in a reasonable amount of time. auto tree_overlay = TreeOperationsUtils::ComputeTreeOverlay(*local_api, *base_tree_info, *other_tree_info, /*disjoint=*/false); REQUIRE(tree_overlay); // Check actual result. auto result_tree_info = CreateNestedTree(kTreeLevels, *local_api, hash_function, {{"foo", "foo"}, {"bar", "bar"}}); REQUIRE(result_tree_info); CHECK(*tree_overlay == *result_tree_info); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/000077500000000000000000000000001516554100600251165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/TARGETS000066400000000000000000000150371516554100600261600ustar00rootroot00000000000000{ "file_system_manager": { "type": ["@", "rules", "CC/test", "test"] , "name": ["file_system_manager"] , "srcs": ["file_system_manager.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/system", "system"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "file_system"] } , "object_cas": { "type": ["@", "rules", "CC/test", "test"] , "name": ["object_cas"] , "srcs": ["object_cas.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_cas"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "file_system"] } , "git_tree": { "type": ["@", "rules", "CC/test", "test"] , "name": ["git_tree"] , "srcs": ["git_tree.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "git_cas"] , ["@", "src", "src/buildtool/file_system", "git_repo"] , ["@", "src", "src/buildtool/file_system", "git_tree"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/utils/cpp", "atomic"] , ["@", "src", "src/utils/cpp", "hex_string"] , ["", "catch-main"] , ["utils", "container_matchers"] , ["utils", "shell_quoting"] ] , "stage": ["test", "buildtool", "file_system"] } , "file_root": { "type": ["@", "rules", "CC/test", "test"] , "name": ["file_root"] , "srcs": ["file_root.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/utils/cpp", "expected"] , ["", "catch-main"] , ["utils", "shell_quoting"] , ["utils", "test_hash_function_type"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "file_system"] } , "directory_entries": { "type": ["@", "rules", "CC/test", "test"] , "name": ["directory_entries"] , "srcs": ["directory_entries.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["", "catch-main"] , ["utils", "shell_quoting"] ] , "stage": ["test", "buildtool", "file_system"] } , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "data/empty_executable" , "data/example_file" , "data/test_repo.bundle" , "data/test_repo_symlinks.bundle" , "data/subdir/nested_file" ] , "stage": ["test", "buildtool", "file_system"] } , "data/test_repo.bundle": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["create_fs_test_git_bundle.sh"] , "outs": ["data/test_repo.bundle"] , "cmds": ["sh create_fs_test_git_bundle.sh"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "data/test_repo_symlinks.bundle": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "deps": ["create_fs_test_git_bundle_symlinks.sh"] , "outs": ["data/test_repo_symlinks.bundle"] , "cmds": ["sh create_fs_test_git_bundle_symlinks.sh"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "git_repo": { "type": ["@", "rules", "CC/test", "test"] , "name": ["git_repo"] , "srcs": ["git_repo.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "git_cas"] , ["@", "src", "src/buildtool/file_system", "git_repo"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/utils/cpp", "atomic"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "hex_string"] , ["", "catch-main"] , ["utils", "shell_quoting"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "file_system"] } , "resolve_symlinks_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["resolve_symlinks_map"] , "srcs": ["resolve_symlinks_map.test.cpp"] , "data": ["test_data"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "git_cas"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/file_system/symlinks", "pragma_special"] , [ "@" , "src" , "src/buildtool/file_system/symlinks" , "resolve_symlinks_map" ] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["", "catch-main"] , ["utils", "shell_quoting"] ] , "stage": ["test", "buildtool", "file_system"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["file_system"] , "deps": [ "directory_entries" , "file_root" , "file_system_manager" , "git_repo" , "git_tree" , "object_cas" , "resolve_symlinks_map" ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/create_fs_test_git_bundle.sh000066400000000000000000000033551516554100600326460ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e if ! command -v git >/dev/null; then echo "git is required to run this script." exit 1 fi # --- # Structure of test_repo: # --- # <--kTreeId: c610db170fbcad5f2d66fe19972495923f3b2536 (tree) # | # +--bar <--kBarId: ba0e162e1c47469e3fe4b393a8bf8c569f302116 (blob) # +--foo <--kFooId: 19102815663d23f8b75a47e7a01965dcdc96468c (blob) # +--baz <--kBazId: 27b32561185c2825150893774953906c6daa6798 (tree) # | +--bar # | +--foo # | # --- # # kCommitId: e4fc610c60716286b98cf51ad0c8f0d50f3aebb5 (commit) # # foo is a regular file # bar is an executable # # Bundle name: test_repo.bundle # # create the folder structure mkdir -p test_repo cd test_repo printf %s "foo" >> foo # no newline printf %s "bar" >> bar # no newline chmod +x bar mkdir -p baz cp foo baz/foo cp bar baz/bar # create the repo git init > /dev/null 2>&1 git checkout -q -b master git config user.name "Nobody" git config user.email "nobody@example.org" git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "First commit" > /dev/null # create the git bundle git bundle create ../data/test_repo.bundle HEAD master just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/create_fs_test_git_bundle_symlinks.sh000066400000000000000000000044211516554100600345720ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e if ! command -v git >/dev/null; then echo "git is required to run this script." exit 1 fi # --- # Structure of test_repo_symlinks: # --- # <--kTreeSymId: 18770dacfe14c15d88450c21c16668e13ab0e7f9 (tree) # | # +--bar <--kBarId: same as in test_repo.bundle (blob) # +--foo <--kFooId: same as in test_repo.bundle (blob) # +--baz_l <-- kBazLinkId: 3f9538666251333f5fa519e01eb267d371ca9c78 (blob) # +--foo_l <-- kFooLinkId: b24736f10d3c60015386047ebc98b4ab63056041 (blob) # +--baz <--kBazSymId: 1868f82682c290f0b1db3cacd092727eef1fa57f (tree) # | +--bar # | +--foo # | +--bar_l <-- kBazBarLinkId: same as kBarId [same content] (blob) # | # --- # # kRootCoomit: 3ecce3f5b19ad7941c6354d65d841590662f33ef (commit) # # foo is a regular file # bar is an executable # # foo_l is a symlink to "baz/foo" # bar_l is a symlink to "bar" (i.e., baz/bar) # baz_l is a symlink to "baz" # # Bundle name: test_repo_symlinks.bundle # # create the folder structure mkdir -p test_repo_symlinks cd test_repo_symlinks printf %s "foo" >> foo # no newline printf %s "bar" >> bar # no newline chmod +x bar mkdir -p baz/ cp foo baz/foo cp bar baz/bar ln -s baz/foo foo_l ln -s bar baz/bar_l ln -s baz baz_l # create the repo git init > /dev/null 2>&1 git checkout -q -b master git config user.name "Nobody" git config user.email "nobody@example.org" git add . GIT_AUTHOR_DATE="1970-01-01T00:00Z" GIT_COMMITTER_DATE="1970-01-01T00:00Z" git commit -m "First commit" > /dev/null # create the git bundle git bundle create ../data/test_repo_symlinks.bundle HEAD master # unlink the symlinks ourselves, to avoid broken symlinks issues on cleanup unlink baz_l unlink baz/bar_l unlink foo_l just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/data/000077500000000000000000000000001516554100600260275ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/data/empty_executable000077500000000000000000000000001516554100600313020ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/data/example_file000066400000000000000000000000511516554100600304000ustar00rootroot00000000000000First line Line 2 Last line with content just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/data/subdir/000077500000000000000000000000001516554100600273175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/data/subdir/nested_file000066400000000000000000000000041516554100600315150ustar00rootroot00000000000000zzz just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/directory_entries.test.cpp000066400000000000000000000233541516554100600323440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo.bundle"}; auto const kBundleSymPath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/file_system"; } [[nodiscard]] auto CreateTestRepo(bool do_checkout = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", do_checkout ? "--branch master " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] auto CreateTestRepoSymlinks(bool do_checkout = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo_symlinks" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", do_checkout ? "--branch master " : "", QuoteForShell(kBundleSymPath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } } // namespace TEST_CASE("Get entries of a directory", "[directory_entries]") { const std::unordered_set reference_entries{ "test_repo.bundle", "test_repo_symlinks.bundle", "empty_executable", "subdir", "example_file"}; const auto dir = std::filesystem::path("test/buildtool/file_system/data"); auto fs_root = FileRoot(); auto dir_entries = fs_root.ReadDirectory(dir); REQUIRE(dir_entries.has_value()); CHECK(dir_entries->ContainsBlob("test_repo.bundle")); CHECK(dir_entries->ContainsBlob("test_repo_symlinks.bundle")); CHECK(dir_entries->ContainsBlob("empty_executable")); CHECK(dir_entries->ContainsBlob("example_file")); { // all the entries returned by FilesIterator are files, // are contained in reference_entries, // and are 4 auto counter = 0; for (const auto& x : dir_entries->FilesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 4); } { // all the entries returned by DirectoriesIterator are not files (e.g., // trees), // are contained in reference_entries, // and are 1 auto counter = 0; for (const auto& x : dir_entries->DirectoriesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK_FALSE(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 1); } } TEST_CASE("Get entries of a git tree", "[directory_entries]") { auto reference_entries = std::unordered_set{"foo", "bar", "baz", ".git"}; auto repo = *CreateTestRepo(true); auto fs_root = FileRoot(); auto dir_entries = fs_root.ReadDirectory(repo); REQUIRE(dir_entries.has_value()); CHECK(dir_entries->ContainsBlob("bar")); CHECK(dir_entries->ContainsBlob("foo")); CHECK_FALSE(dir_entries->ContainsBlob("baz")); { // all the entries returned by FilesIterator are files, // are contained in reference_entries, // and are 2 (foo, and bar) auto counter = 0; for (const auto& x : dir_entries->FilesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } { // all the entries returned by DirectoriesIterator are not files (e.g., // trees), // are contained in reference_entries, // and are 2 (baz, and .git) auto counter = 0; for (const auto& x : dir_entries->DirectoriesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK_FALSE(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } } TEST_CASE("Get entries of an empty directory", "[directory_entries]") { // CreateDirectoryEntriesMap returns // FileRoot::DirectoryEntries{FileRoot::DirectoryEntries::pairs_t{}} in case // of a missing directory, which represents an empty directory auto dir_entries = FileRoot::DirectoryEntries{FileRoot::DirectoryEntries::pairs_t{}}; // of course, no files should be there CHECK_FALSE(dir_entries.ContainsBlob("test_repo.bundle")); { // FilesIterator should be an empty range auto counter = 0; for (const auto& x : dir_entries.FilesIterator()) { CHECK_FALSE(dir_entries.ContainsBlob(x)); // should never be called ++counter; } CHECK(counter == 0); } { // DirectoriesIterator should be an empty range auto counter = 0; for (const auto& x : dir_entries.DirectoriesIterator()) { CHECK(dir_entries.ContainsBlob(x)); // should never be called ++counter; } CHECK(counter == 0); } } TEST_CASE("Get ignore-special entries of a git tree with symlinks", "[directory_entries]") { auto reference_entries = std::unordered_set{"foo", "bar", "baz", ".git"}; auto repo = *CreateTestRepoSymlinks(true); auto fs_root = FileRoot(/*ignore_special=*/true); auto dir_entries = fs_root.ReadDirectory(repo); REQUIRE(dir_entries.has_value()); CHECK(dir_entries->ContainsBlob("bar")); CHECK(dir_entries->ContainsBlob("foo")); CHECK_FALSE(dir_entries->ContainsBlob("baz")); { // all the entries returned by FilesIterator are files, // are contained in reference_entries, // and are 2 (foo, and bar) auto counter = 0; for (const auto& x : dir_entries->FilesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } { // all the entries returned by DirectoriesIterator are not files (e.g., // trees), // are contained in reference_entries, // and are 2 (baz, and .git) auto counter = 0; for (const auto& x : dir_entries->DirectoriesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK_FALSE(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } } TEST_CASE("Get entries of a git tree with symlinks", "[directory_entries]") { auto reference_entries = std::unordered_set{ "foo", "bar", "baz", "foo_l", "baz_l", ".git"}; auto repo = *CreateTestRepoSymlinks(true); auto fs_root = FileRoot(/*ignore_special=*/false); auto dir_entries = fs_root.ReadDirectory(repo); REQUIRE(dir_entries.has_value()); CHECK(dir_entries->ContainsBlob("bar")); CHECK(dir_entries->ContainsBlob("foo")); CHECK_FALSE(dir_entries->ContainsBlob("baz")); CHECK(dir_entries->ContainsBlob("foo_l")); CHECK(dir_entries->ContainsBlob("baz_l")); { // all the entries returned by FilesIterator are files, // are contained in reference_entries, // and are 2 (foo, and bar) auto counter = 0; for (const auto& x : dir_entries->FilesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } { // all the entries returned by SymlinksIterator are symlinks, // are contained in reference_entries, // and are 2 (foo_l, and baz_l) auto counter = 0; for (const auto& x : dir_entries->SymlinksIterator()) { REQUIRE(reference_entries.contains(x)); CHECK(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } { // all the entries returned by DirectoriesIterator are not files (e.g., // trees), // are contained in reference_entries, // and are 2 (baz, and .git) auto counter = 0; for (const auto& x : dir_entries->DirectoriesIterator()) { REQUIRE(reference_entries.contains(x)); CHECK_FALSE(dir_entries->ContainsBlob(x)); ++counter; } CHECK(counter == 2); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/file_root.test.cpp000066400000000000000000000421561516554100600305720ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/file_root.hpp" #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundleSymPath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kTreeSymId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kFooIdGitSha1 = std::string{"19102815663d23f8b75a47e7a01965dcdc96468c"}; auto const kFooIdSha256 = std::string{ "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}; auto constexpr kFooContentLength = std::string_view{"foo"}.size(); auto const kBarIdGitSha1 = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kBarIdSha256 = std::string{ "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9"}; auto constexpr kBarContentLength = std::string_view{"bar"}.size(); [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/file_system"; } [[nodiscard]] auto CreateTestRepoSymlinks(bool do_checkout = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo_symlinks" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", do_checkout ? "--branch master " : "", QuoteForShell(kBundleSymPath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } void TestFileRootReadCommonFiles(FileRoot const& root) { REQUIRE(root.Exists("foo")); REQUIRE(root.IsFile("foo")); auto foo = root.ReadContent("foo"); REQUIRE(foo); CHECK(*foo == "foo"); REQUIRE(root.Exists("bar")); REQUIRE(root.IsFile("bar")); auto bar = root.ReadContent("bar"); REQUIRE(bar); CHECK(*bar == "bar"); REQUIRE(root.Exists("baz")); REQUIRE(root.IsDirectory("baz")); // Check subdir REQUIRE(root.Exists("baz/foo")); REQUIRE(root.IsFile("baz/foo")); auto bazfoo = root.ReadContent("baz/foo"); REQUIRE(bazfoo); CHECK(*bazfoo == "foo"); REQUIRE(root.Exists("baz/bar")); REQUIRE(root.IsFile("baz/bar")); auto bazbar = root.ReadContent("baz/bar"); REQUIRE(bazbar); CHECK(*bazbar == "bar"); } void TestFileRootReadFilesOnly(FileRoot const& root) { // Check common files TestFileRootReadCommonFiles(root); // Check symlinks are missing CHECK_FALSE(root.Exists("baz_l")); CHECK_FALSE(root.Exists("foo_l")); CHECK_FALSE(root.Exists("baz/bar_l")); } void TestFileRootReadFilesAndSymlinks(FileRoot const& root) { // Check common files TestFileRootReadCommonFiles(root); // Check symlinks CHECK(root.Exists("baz_l")); CHECK(root.Exists("foo_l")); CHECK(root.Exists("baz/bar_l")); } void TestFileRootReadEntries(FileRoot const& root, std::string const& path, bool has_baz, bool with_symlinks) { REQUIRE(root.Exists(path)); REQUIRE(root.IsDirectory(path)); auto entries = root.ReadDirectory(path); REQUIRE(entries.has_value()); CHECK_FALSE(entries->Empty()); CHECK(entries->ContainsBlob("foo")); CHECK(entries->ContainsBlob("bar")); if (has_baz) { CHECK_FALSE(entries->ContainsBlob("baz")); CHECK(with_symlinks == entries->ContainsBlob("baz_l")); CHECK(with_symlinks == entries->ContainsBlob("foo_l")); } else { CHECK(with_symlinks == entries->ContainsBlob("bar_l")); } CHECK_FALSE(entries->ContainsBlob("does_not_exist")); } void TestFileRootReadDirectory(FileRoot const& root, bool with_symlinks) { TestFileRootReadEntries(root, ".", true, with_symlinks); TestFileRootReadEntries(root, "baz", false, with_symlinks); } void TestFileRootReadBlobType(FileRoot const& root) { auto foo_type = root.BlobType("baz/foo"); REQUIRE(foo_type); CHECK(*foo_type == ObjectType::File); auto bar_type = root.BlobType("baz/bar"); REQUIRE(bar_type); CHECK(*bar_type == ObjectType::Executable); CHECK_FALSE(root.BlobType("baz")); CHECK_FALSE(root.BlobType("does_not_exist")); // Check subdir REQUIRE(root.Exists("baz/foo")); REQUIRE(root.IsFile("baz/foo")); auto bazfoo = root.ReadContent("baz/foo"); REQUIRE(bazfoo); CHECK(*bazfoo == "foo"); REQUIRE(root.Exists("baz/bar")); REQUIRE(root.IsFile("baz/bar")); auto bazbar = root.ReadContent("baz/bar"); REQUIRE(bazbar); CHECK(*bazbar == "bar"); } } // namespace TEST_CASE("Creating file root", "[file_root]") { SECTION("local root") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); CHECK(FileRoot{*root_path}.Exists(".")); CHECK_FALSE( FileRoot{std::filesystem::path{"does_not_exist"}}.Exists(".")); } SECTION("git root") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId); REQUIRE(root); CHECK(root->Exists(".")); CHECK_FALSE(FileRoot::FromGit( &storage_config.Get(), "does_not_exist", kTreeSymId)); } SECTION("local root ignore-special") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); CHECK(FileRoot{*root_path, /*ignore_special=*/true}.Exists(".")); CHECK_FALSE(FileRoot{std::filesystem::path{"does_not_exist"}, /*ignore_special=*/true} .Exists(".")); } SECTION("git root ignore-special") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId, /*ignore_special=*/true); REQUIRE(root); CHECK(root->Exists(".")); CHECK_FALSE(FileRoot::FromGit(&storage_config.Get(), "does_not_exist", kTreeSymId, /*ignore_special=*/true)); } } TEST_CASE("Reading files", "[file_root]") { SECTION("local root") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadFilesAndSymlinks(FileRoot{*root_path}); } SECTION("git root") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId); REQUIRE(root); TestFileRootReadFilesAndSymlinks(*root); } SECTION("local root ignore-special") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadFilesOnly( FileRoot{*root_path, /*ignore_special=*/true}); } SECTION("git root ignore-special") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId, /*ignore_special=*/true); REQUIRE(root); TestFileRootReadFilesOnly(*root); } } TEST_CASE("Reading directories", "[file_root]") { SECTION("local root") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadDirectory(FileRoot{*root_path}, /*with_symlinks=*/true); } SECTION("git root") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId); REQUIRE(root); TestFileRootReadDirectory(*root, /*with_symlinks=*/true); } SECTION("local root ignore-special") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadDirectory(FileRoot{*root_path, /*ignore_special=*/true}, /*with_symlinks=*/false); } SECTION("git root ignore-special") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId, /*ignore_special=*/true); REQUIRE(root); TestFileRootReadDirectory(*root, /*with_symlinks=*/false); } } TEST_CASE("Reading blobs", "[file_root]") { SECTION("local root") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); CHECK_FALSE(FileRoot{*root_path}.ReadBlob(kFooIdGitSha1)); } SECTION("git root") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId); REQUIRE(root); auto foo = root->ReadBlob(kFooIdGitSha1); REQUIRE(foo); CHECK(*foo == "foo"); CHECK_FALSE(root->ReadBlob("does_not_exist")); } SECTION("local root ignore-special") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); CHECK_FALSE(FileRoot{*root_path, /*ignore_special=*/true}.ReadBlob( kFooIdGitSha1)); } SECTION("git root ignore-special") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId, /*ignore_special=*/true); REQUIRE(root); auto foo = root->ReadBlob(kFooIdGitSha1); REQUIRE(foo); CHECK(*foo == "foo"); CHECK_FALSE(root->ReadBlob("does_not_exist")); } } TEST_CASE("Reading blob type", "[file_root]") { SECTION("local root") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadBlobType(FileRoot{*root_path}); } SECTION("git root") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId); REQUIRE(root); TestFileRootReadBlobType(*root); } SECTION("local root ignore-special") { auto root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); TestFileRootReadBlobType(FileRoot{*root_path, /*ignore_special=*/true}); } SECTION("git root ignore-special") { auto repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto root = FileRoot::FromGit(&storage_config.Get(), *repo_path, kTreeSymId, /*ignore_special=*/true); REQUIRE(root); TestFileRootReadBlobType(*root); } } static void CheckLocalRoot(HashFunction::Type hash_type, bool ignore_special) noexcept; static void CheckGitRoot(HashFunction::Type hash_type, bool ignore_special) noexcept; TEST_CASE("Creating artifact descriptions", "[file_root]") { auto const hash_type = TestHashType::ReadFromEnvironment(); SECTION("local root") { CheckLocalRoot(hash_type, /*ignore_special=*/false); } SECTION("git root") { CheckGitRoot(hash_type, /*ignore_special=*/false); } SECTION("local root ignore-special") { CheckLocalRoot(hash_type, /*ignore_special=*/true); } SECTION("git root ignore-special") { CheckGitRoot(hash_type, /*ignore_special=*/true); } } static void CheckLocalRoot(HashFunction::Type hash_type, bool ignore_special) noexcept { auto const root_path = CreateTestRepoSymlinks(true); REQUIRE(root_path); auto const root = FileRoot{*root_path, ignore_special}; auto const desc = root.ToArtifactDescription(hash_type, "baz/foo", "repo"); REQUIRE(desc); CHECK(*desc == ArtifactDescription::CreateLocal( std::filesystem::path{"baz/foo"}, "repo")); CHECK(root.ToArtifactDescription(hash_type, "does_not_exist", "repo")); } static void CheckGitRoot(HashFunction::Type hash_type, bool ignore_special) noexcept { auto const repo_path = CreateTestRepoSymlinks(false); REQUIRE(repo_path); auto const storage_config = TestStorageConfig::Create(); auto const root = FileRoot::FromGit( &storage_config.Get(), *repo_path, kTreeSymId, ignore_special); REQUIRE(root); auto const foo = root->ToArtifactDescription(hash_type, "baz/foo", "repo"); REQUIRE(foo); if (not ProtocolTraits::IsNative(hash_type)) { auto const digest = ArtifactDigestFactory::Create(hash_type, kFooIdSha256, kFooContentLength, /*is_tree=*/false); REQUIRE(digest); CHECK(*foo == ArtifactDescription::CreateKnown(*digest, ObjectType::File)); } else { auto const digest = ArtifactDigestFactory::Create(hash_type, kFooIdGitSha1, kFooContentLength, /*is_tree=*/false); REQUIRE(digest); CHECK(*foo == ArtifactDescription::CreateKnown( *digest, ObjectType::File, "repo")); } auto const bar = root->ToArtifactDescription(hash_type, "baz/bar", "repo"); REQUIRE(bar); if (not ProtocolTraits::IsNative(hash_type)) { auto const digest = ArtifactDigestFactory::Create(hash_type, kBarIdSha256, kBarContentLength, /*is_tree=*/false); REQUIRE(digest); CHECK(*bar == ArtifactDescription::CreateKnown(*digest, ObjectType::Executable)); } else { auto const digest = ArtifactDigestFactory::Create(hash_type, kBarIdGitSha1, kBarContentLength, /*is_tree=*/false); REQUIRE(digest); CHECK(*bar == ArtifactDescription::CreateKnown( *digest, ObjectType::Executable, "repo")); } CHECK_FALSE(root->ToArtifactDescription(hash_type, "baz", "repo")); CHECK_FALSE( root->ToArtifactDescription(hash_type, "does_not_exist", "repo")); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/file_system_manager.test.cpp000066400000000000000000000751731516554100600326320ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/file_system_manager.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/generators/catch_generators_all.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/system/system.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" class CopyFileFixture { public: CopyFileFixture() noexcept { REQUIRE(FileSystemManager::CreateDirectory(to_.parent_path())); } CopyFileFixture(CopyFileFixture const&) = delete; CopyFileFixture(CopyFileFixture&&) = delete; ~CopyFileFixture() noexcept { CHECK(std::filesystem::remove(to_)); } auto operator=(CopyFileFixture const&) -> CopyFileFixture& = delete; auto operator=(CopyFileFixture&&) -> CopyFileFixture& = delete; std::filesystem::path const from_{ "test/buildtool/file_system/data/example_file"}; std::filesystem::path const to_{"./tmp-CopyFile/copied_file"}; }; class WriteFileFixture { public: WriteFileFixture() noexcept { REQUIRE(FileSystemManager::CreateDirectory(root_dir_)); } WriteFileFixture(WriteFileFixture const&) = delete; WriteFileFixture(WriteFileFixture&&) = delete; ~WriteFileFixture() noexcept { CHECK(std::filesystem::remove(file_path_)); } auto operator=(WriteFileFixture const&) -> WriteFileFixture& = delete; auto operator=(WriteFileFixture&&) -> WriteFileFixture& = delete; std::filesystem::path const relative_path_parent_{ GENERATE(as{}, ".", "level0", "level0/level1", "a/b/c/d", "./a/../e")}; std::filesystem::path const root_dir_{"./tmp-RemoveFile"}; std::filesystem::path const file_path_{root_dir_ / relative_path_parent_ / "file"}; }; class SymlinkTestsFixture { public: SymlinkTestsFixture() noexcept { REQUIRE(FileSystemManager::CreateDirectory(root_dir_)); create_files(); create_symlinks(); } SymlinkTestsFixture(SymlinkTestsFixture const&) = delete; SymlinkTestsFixture(SymlinkTestsFixture&&) = delete; ~SymlinkTestsFixture() noexcept { CHECK(std::filesystem::remove_all(root_dir_)); } auto operator=(SymlinkTestsFixture const&) -> SymlinkTestsFixture& = delete; auto operator=(SymlinkTestsFixture&&) -> SymlinkTestsFixture& = delete; std::filesystem::path const root_dir_{"./tmp-Symlinks"}; using filetree_t = std::unordered_map; filetree_t const kExpected = {{"foo", ObjectType::File}, {"baz", ObjectType::Tree}, {"baz/foo", ObjectType::File}, {"bazz", ObjectType::Tree}, {"bazz/baz", ObjectType::Tree}, {"bazz/baz/foo", ObjectType::File}}; struct LinkInfo { std::string to; std::string link; bool resolvesToExisting; bool isNonUpwards; }; std::vector const kSymExpected = { {.to = "baz", .link = "baz_l", .resolvesToExisting = true, .isNonUpwards = true}, {.to = "../foo", .link = "baz/foo_l", .resolvesToExisting = true, .isNonUpwards = false}, {.to = "baz/foo_l", .link = "bar_l", .resolvesToExisting = true, .isNonUpwards = true}, {.to = "does_not_exist", .link = "baz/non_existing_l", .resolvesToExisting = false, .isNonUpwards = true}, {.to = "non_existing_l", .link = "baz/non_existing_indirect_l", .resolvesToExisting = false, .isNonUpwards = true}, {.to = "baz/../../does_not_exist", .link = "non_existing_sneaky_l", .resolvesToExisting = false, .isNonUpwards = false}}; // distinct dir entries size_t const num_entries_{12U}; // distinct dir entries after removing all subdirs named "baz" size_t const num_root_file_entries_{5U}; void create_files() { for (auto const& [path, type] : kExpected) { switch (type) { case ObjectType::File: { if (not FileSystemManager::WriteFile("", root_dir_ / path)) { Logger::Log(LogLevel::Error, "Could not create test file at path {}", (root_dir_ / path).string()); std::exit(1); }; } break; case ObjectType::Tree: { if (not FileSystemManager::CreateDirectory(root_dir_ / path)) { Logger::Log(LogLevel::Error, "Could not create test dir at path {}", (root_dir_ / path).string()); std::exit(1); }; } break; default: { Logger::Log(LogLevel::Error, "File system failure in creating test dir"); std::exit(1); } } } } void create_symlinks() { for (auto const& link_info : kSymExpected) { if (not FileSystemManager::CreateSymlink( link_info.to, root_dir_ / link_info.link)) { Logger::Log( LogLevel::Error, "File system failure in creating symlink at path {}", (root_dir_ / link_info.link).string()); std::exit(1); } } } }; namespace { namespace fs = std::filesystem; constexpr auto kFilePerms = fs::perms::owner_read | fs::perms::group_read | fs::perms::others_read; constexpr auto kInstalledPerms = fs::perms::owner_write; constexpr auto kExecPerms = fs::perms::owner_exec | fs::perms::group_exec | fs::perms::others_exec; auto HasFilePermissions(fs::path const& path) noexcept -> bool { try { return fs::status(path).permissions() == kFilePerms; } catch (...) { return false; } } auto HasInstalledFilePermissions(fs::path const& path) noexcept -> bool { try { return fs::status(path).permissions() == (kFilePerms | kInstalledPerms); } catch (...) { return false; } } auto HasExecutablePermissions(fs::path const& path) noexcept -> bool { try { return fs::status(path).permissions() == (kFilePerms | kExecPerms); } catch (...) { return false; } } auto HasInstalledExecutablePermissions(fs::path const& path) noexcept -> bool { try { return fs::status(path).permissions() == (kFilePerms | kExecPerms | kInstalledPerms); } catch (...) { return false; } } auto HasEpochTime(fs::path const& path) noexcept -> bool { try { return fs::last_write_time(path) == System::GetPosixEpoch(); } catch (...) { return false; } } } // namespace TEST_CASE("Exists", "[file_system]") { CHECK(FileSystemManager::Exists("test/buildtool/file_system/data/")); CHECK(FileSystemManager::Exists( "test/buildtool/file_system/data/example_file")); CHECK(FileSystemManager::Exists( "test/buildtool/file_system/data/empty_executable")); } TEST_CASE("IsFile", "[file_system]") { CHECK(FileSystemManager::IsFile( "test/buildtool/file_system/data/example_file")); CHECK(FileSystemManager::IsFile( "test/buildtool/file_system/data/empty_executable")); CHECK_FALSE(FileSystemManager::IsFile("test/buildtool/file_system/data/")); } TEST_CASE("IsExecutable", "[file_system]") { CHECK(FileSystemManager::IsExecutable( "test/buildtool/file_system/data/empty_executable")); CHECK_FALSE(FileSystemManager::IsExecutable( "test/buildtool/file_system/data/example_file")); CHECK_FALSE( FileSystemManager::IsExecutable("test/buildtool/file_system/data/")); } TEST_CASE("IsDirectory", "[file_system]") { CHECK(FileSystemManager::IsDirectory("test/buildtool/file_system/data")); CHECK_FALSE(FileSystemManager::IsDirectory( "test/buildtool/file_system/data/example_file")); CHECK_FALSE(FileSystemManager::IsDirectory( "test/buildtool/file_system/data/empty_executable")); } TEST_CASE("Type", "[file_system]") { auto const type_file = FileSystemManager::Type("test/buildtool/file_system/data/example_file"); REQUIRE(type_file); CHECK(*type_file == ObjectType::File); auto const type_exec = FileSystemManager::Type( "test/buildtool/file_system/data/empty_executable"); REQUIRE(type_exec); CHECK(*type_exec == ObjectType::Executable); auto const type_dir = FileSystemManager::Type("test/buildtool/file_system/data/"); REQUIRE(type_dir); CHECK(*type_dir == ObjectType::Tree); } TEST_CASE("CreateDirectory", "[file_system]") { auto const dir = GENERATE(as{}, "level0", "level0/level1", "a/b/c/d", "./a/../e"); CHECK(FileSystemManager::CreateDirectory(dir)); CHECK(FileSystemManager::IsDirectory(dir)); // If we have created the directory already, CreateDirectory() returns true // and the state of things doesn't change CHECK(FileSystemManager::CreateDirectory(dir)); CHECK(FileSystemManager::IsDirectory(dir)); } TEST_CASE("ChangeDirectory", "[file_system]") { auto const starting_dir = FileSystemManager::GetCurrentDirectory(); auto const new_dir = GENERATE(as{}, "level0", "level0/level1", "a/b/c/d", "./a/../e"); REQUIRE(FileSystemManager::CreateDirectory(new_dir)); { auto anchor = FileSystemManager::ChangeDirectory(new_dir); CHECK(std::filesystem::equivalent( starting_dir / new_dir, FileSystemManager::GetCurrentDirectory())); } CHECK(starting_dir == FileSystemManager::GetCurrentDirectory()); } TEST_CASE("ReadFile", "[file_system]") { SECTION("Existing file") { std::string const expected_content{"test\n"}; std::filesystem::path file{"./tmp-ReadFile/file"}; REQUIRE(FileSystemManager::CreateDirectory(file.parent_path())); std::ofstream writer{file}; writer << expected_content; writer.close(); auto const content = FileSystemManager::ReadFile(file); CHECK(content.has_value()); CHECK(content == expected_content); } SECTION("Non-existing file") { std::filesystem::path file{ "test/buildtool/file_system/data/this_file_does_not_exist"}; REQUIRE(not FileSystemManager::Exists(file)); auto const content = FileSystemManager::ReadFile(file); CHECK_FALSE(content.has_value()); } } TEST_CASE_METHOD(CopyFileFixture, "CopyFile", "[file_system]") { auto run_test = [&](bool fd_less) { // Copy file was successful CHECK(FileSystemManager::CopyFile(from_, to_, fd_less)); // file exists CHECK(FileSystemManager::IsFile(to_)); // Contents are equal auto const content_from = FileSystemManager::ReadFile(from_); CHECK(content_from.has_value()); auto const content_to = FileSystemManager::ReadFile(to_); CHECK(content_to.has_value()); CHECK(content_from == content_to); }; SECTION("direct") { run_test(false); } SECTION("fd_less") { run_test(true); } } TEST_CASE("CopyFile equivalent", "[file_system]") { auto storage_config = TestStorageConfig::Create(); auto temp_dir = storage_config.Get().CreateTypedTmpDir("test"); REQUIRE(temp_dir); auto const nested = temp_dir->GetPath() / "dir"; auto const source = nested / "copy-self"; auto const target = nested / "../dir/copy-self"; REQUIRE(FileSystemManager::CreateDirectory(nested)); REQUIRE(FileSystemManager::CreateFile(source)); REQUIRE(FileSystemManager::IsFile(source)); REQUIRE(FileSystemManager::IsFile(target)); // Paths are different, but equivalent: REQUIRE(source != target); REQUIRE(std::filesystem::equivalent(source, target)); CHECK(FileSystemManager::CopyFile(source, target)); CHECK(FileSystemManager::IsFile(source)); CHECK(FileSystemManager::IsFile(target)); } TEST_CASE_METHOD(CopyFileFixture, "CopyFileAs", "[file_system]") { SECTION("as file") { auto run_test = [&](bool fd_less) { // Copy as file was successful CHECK(FileSystemManager::CopyFileAs( from_, to_, ObjectType::File, fd_less)); // file exists CHECK(FileSystemManager::IsFile(to_)); CHECK(not FileSystemManager::IsExecutable(to_)); // Contents are equal auto const content_from = FileSystemManager::ReadFile(from_); CHECK(content_from.has_value()); auto const content_to = FileSystemManager::ReadFile(to_); CHECK(content_to.has_value()); CHECK(content_from == content_to); // permissions should be 0444 CHECK(HasFilePermissions(to_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(to_)); } }; SECTION("direct") { run_test(false); } SECTION("fd_less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd_less with epoch") { run_test.template operator()(true); } } SECTION("as installed file") { auto run_test = [&](bool fd_less) { // Copy as file was successful CHECK(FileSystemManager::CopyFileAs( from_, to_, ObjectType::File, fd_less)); // file exists CHECK(FileSystemManager::IsFile(to_)); CHECK(not FileSystemManager::IsExecutable(to_)); // Contents are equal auto const content_from = FileSystemManager::ReadFile(from_); CHECK(content_from.has_value()); auto const content_to = FileSystemManager::ReadFile(to_); CHECK(content_to.has_value()); CHECK(content_from == content_to); // permissions should be 0644 CHECK(HasInstalledFilePermissions(to_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(to_)); } }; SECTION("direct") { run_test(false); } SECTION("fd_less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd_less with epoch") { run_test.template operator()(true); } } SECTION("as executable") { auto run_test = [&](bool fd_less) { // Copy as file was successful CHECK(FileSystemManager::CopyFileAs( from_, to_, ObjectType::Executable, fd_less)); // file exists CHECK(FileSystemManager::IsExecutable(to_)); // Contents are equal auto const content_from = FileSystemManager::ReadFile(from_); CHECK(content_from.has_value()); auto const content_to = FileSystemManager::ReadFile(to_); CHECK(content_to.has_value()); CHECK(content_from == content_to); // permissions should be 0555 CHECK(HasExecutablePermissions(to_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(to_)); } }; SECTION("direct") { run_test(false); } SECTION("fd_less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd_less with epoch") { run_test.template operator()(true); } } SECTION("as installed executable") { auto run_test = [&](bool fd_less) { // Copy as file was successful CHECK(FileSystemManager::CopyFileAs( from_, to_, ObjectType::Executable, fd_less)); // file exists CHECK(FileSystemManager::IsExecutable(to_)); // Contents are equal auto const content_from = FileSystemManager::ReadFile(from_); CHECK(content_from.has_value()); auto const content_to = FileSystemManager::ReadFile(to_); CHECK(content_to.has_value()); CHECK(content_from == content_to); // permissions should be 0755 CHECK(HasInstalledExecutablePermissions(to_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(to_)); } }; SECTION("direct") { run_test(false); } SECTION("fd_less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd_less with epoch") { run_test.template operator()(true); } } } TEST_CASE("RemoveFile", "[file_system]") { SECTION("Existing file") { std::filesystem::path from{ "test/buildtool/file_system/data/example_file"}; std::filesystem::path to{"./tmp-RemoveFile/copied_file"}; REQUIRE(FileSystemManager::CreateDirectory(to.parent_path())); CHECK(FileSystemManager::CopyFile(from, to)); CHECK(FileSystemManager::Exists(to)); CHECK(FileSystemManager::RemoveFile(to)); CHECK(not FileSystemManager::Exists(to)); } SECTION("Non-existing file") { std::filesystem::path file{ "test/buildtool/file_system/data/" "this_file_does_not_exist_neither"}; CHECK(not FileSystemManager::Exists(file)); CHECK(FileSystemManager::RemoveFile(file)); // nothing to delete } SECTION("Existing but not file") { std::filesystem::path dir{"./tmp-RemoveFile/dir"}; CHECK(FileSystemManager::CreateDirectory(dir)); CHECK(not FileSystemManager::RemoveFile(dir)); CHECK(FileSystemManager::Exists(dir)); } } TEST_CASE_METHOD(WriteFileFixture, "WriteFile", "[file_system]") { std::string const content{"This are the contents\nof the file.\n"}; auto run_test = [&](bool fd_less) { CHECK(FileSystemManager::WriteFile(content, file_path_, fd_less)); CHECK(FileSystemManager::IsDirectory(file_path_.parent_path())); CHECK(FileSystemManager::IsFile(file_path_)); auto const written_content = FileSystemManager::ReadFile(file_path_); CHECK(written_content.has_value()); CHECK(written_content == content); }; SECTION("direct") { run_test(false); } SECTION("fd-less") { run_test(true); } } TEST_CASE_METHOD(WriteFileFixture, "WriteFileAs", "[file_system]") { SECTION("as a file") { std::string const content{"This are the contents\nof the file.\n"}; auto run_test = [&](bool fd_less) { CHECK(FileSystemManager::WriteFileAs( content, file_path_, ObjectType::File, fd_less)); CHECK(FileSystemManager::IsDirectory(file_path_.parent_path())); CHECK(FileSystemManager::IsFile(file_path_)); CHECK(not FileSystemManager::IsExecutable(file_path_)); auto const written_content = FileSystemManager::ReadFile(file_path_); CHECK(written_content.has_value()); CHECK(written_content == content); // permissions should be 0444 CHECK(HasFilePermissions(file_path_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(file_path_)); } }; SECTION("direct") { run_test(false); } SECTION("fd-less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd-less with epoch") { run_test.template operator()(true); } } SECTION("as an executable") { std::string const content{"\n"}; auto run_test = [&](bool fd_less) { CHECK(FileSystemManager::WriteFileAs( content, file_path_, ObjectType::Executable, fd_less)); CHECK(FileSystemManager::IsDirectory(file_path_.parent_path())); CHECK(FileSystemManager::IsExecutable(file_path_)); auto const written_content = FileSystemManager::ReadFile(file_path_); CHECK(written_content.has_value()); CHECK(written_content == content); // permissions should be 0555 CHECK(HasExecutablePermissions(file_path_)); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(file_path_)); } }; SECTION("direct") { run_test(false); } SECTION("fd-less") { run_test(true); } SECTION("direct with epoch") { run_test.template operator()(false); } SECTION("fd-less with epoch") { run_test.template operator()(true); } } } TEST_CASE("FileSystemManager", "[file_system]") { // test file and test file content with newline and null characters std::filesystem::path test_file{"test/file"}; std::filesystem::path copy_file{"test/copy"}; std::string test_content; test_content += "test1"; test_content += '\n'; test_content += '\0'; test_content += "test2"; CHECK(FileSystemManager::IsRelativePath(test_file)); CHECK(not FileSystemManager::IsAbsolutePath(test_file)); // create parent directory REQUIRE(FileSystemManager::CreateDirectory(test_file.parent_path())); // scope to test RAII "DirectoryAnchor" (should restore CWD on destruction) { // change directory and obtain DirectoryAnchor auto anchor = FileSystemManager::ChangeDirectory(test_file.parent_path()); std::ofstream file{test_file.filename()}; REQUIRE(file); file << test_content; file.close(); // check if file exists REQUIRE(FileSystemManager::IsFile(test_file.filename())); } // restore CWD to parent path // check if file exists with full path REQUIRE(FileSystemManager::IsFile(test_file)); // read file content and compare with input above auto const file_content = FileSystemManager::ReadFile(test_file); REQUIRE(file_content.has_value()); CHECK(file_content == test_content); // copy file without 'overwrite' CHECK(FileSystemManager::CopyFile(test_file, copy_file, /*fd_less=*/false, std::filesystem::copy_options::none)); // copy file with 'overwrite' CHECK(FileSystemManager::CopyFile(copy_file, test_file)); // remove files and verify removal CHECK(FileSystemManager::RemoveFile(test_file)); CHECK(not FileSystemManager::IsFile(test_file)); CHECK(FileSystemManager::RemoveFile(copy_file)); CHECK(not FileSystemManager::IsFile(copy_file)); } TEST_CASE("CreateFileHardlink", "[file_system]") { std::filesystem::path to{"./tmp-CreateFileHardlink/linked_file"}; REQUIRE(FileSystemManager::CreateDirectory(to.parent_path())); SECTION("Existing file") { std::filesystem::path from{"example_file"}; { std::ofstream f{from}; f << "foo"; } CHECK(FileSystemManager::CreateFileHardlink(from, to)); CHECK(FileSystemManager::Exists(to)); CHECK_FALSE(FileSystemManager::CreateFileHardlink(from, to)); CHECK(FileSystemManager::Exists(to)); CHECK(FileSystemManager::RemoveFile(to)); CHECK(not FileSystemManager::Exists(to)); } SECTION("Non-existing file") { std::filesystem::path from{ "test/buildtool/file_system/data/this_file_does_not_exist"}; CHECK_FALSE(FileSystemManager::CreateFileHardlink(from, to)); CHECK_FALSE(FileSystemManager::Exists(to)); } SECTION("Existing but not file") { std::filesystem::path from{"./tmp-CreateFileHardlink/dir"}; CHECK(FileSystemManager::CreateDirectory(from)); CHECK_FALSE(FileSystemManager::CreateFileHardlink(from, to)); CHECK_FALSE(FileSystemManager::Exists(to)); } } TEST_CASE("CopyDirectoryImpl", "[file_system]") { std::filesystem::path to{"./tmp-CreateDirCopy/tmp-dir"}; REQUIRE(FileSystemManager::CreateDirectory(to.parent_path())); CHECK(FileSystemManager::CreateDirectory("a/b/c/d")); CHECK(FileSystemManager::IsDirectory("a/b/c/d")); CHECK(FileSystemManager::WriteFile("boo", "a/bb.txt")); // Test copy CHECK(FileSystemManager::CopyDirectoryImpl("a", to)); // Result should be in tmp-dir now CHECK(FileSystemManager::IsDirectory(to)); CHECK(FileSystemManager::IsDirectory(to / "b")); CHECK(FileSystemManager::IsDirectory(to / "b/c")); CHECK(FileSystemManager::IsFile(to / "bb.txt")); } TEST_CASE_METHOD(CopyFileFixture, "CreateFileHardlinkAs", "[file_system]") { auto set_perm = [&](bool is_executable) { auto const content = FileSystemManager::ReadFile(from_); REQUIRE(content); REQUIRE(FileSystemManager::RemoveFile(from_)); REQUIRE(FileSystemManager::WriteFileAs( *content, from_, is_executable ? ObjectType::Executable : ObjectType::File)); }; auto run_test = [&](bool is_executable) { // Hard link creation was successful CHECK(FileSystemManager::CreateFileHardlinkAs( from_, to_, is_executable ? ObjectType::Executable : ObjectType::File)); // file exists CHECK(is_executable == FileSystemManager::IsExecutable(to_)); // executables are special permission files CHECK(FileSystemManager::IsFile(to_)); // permissions should be 0555 or 0444 CHECK((is_executable ? HasExecutablePermissions(to_) : HasFilePermissions(to_))); if constexpr (kSetEpochTime) { CHECK(HasEpochTime(to_)); } }; SECTION("as file") { SECTION("from file") { set_perm(false); run_test(false); } SECTION("from executable") { set_perm(true); run_test(false); } } SECTION("as executable") { SECTION("from file") { set_perm(false); run_test(true); } SECTION("from executable") { set_perm(true); run_test(true); } } SECTION("as file with epoch") { SECTION("from file") { set_perm(false); run_test.template operator()(false); } SECTION("from executable") { set_perm(true); run_test.template operator()(false); } } SECTION("as executable with epoch") { SECTION("from file") { set_perm(false); run_test.template operator()(true); } SECTION("from executable") { set_perm(true); run_test.template operator()(true); } } } TEST_CASE_METHOD(SymlinkTestsFixture, "Symlinks", "[file_system]") { CHECK(std::filesystem::is_directory(root_dir_ / "baz")); CHECK(std::filesystem::is_symlink(root_dir_ / "baz_l")); CHECK_FALSE(std::filesystem::is_directory( std::filesystem::symlink_status(root_dir_ / "baz_l"))); auto i = GENERATE(range(0U, 6U /* kSymExpected.size() */)); SECTION(fmt::format("Non-upwards symlinks - entry {}", i)) { CHECK(FileSystemManager::IsNonUpwardsSymlink(root_dir_ / kSymExpected[i].link) == kSymExpected[i].isNonUpwards); } SECTION(fmt::format("Resolve symlinks - entry {}", i)) { auto path = root_dir_ / kSymExpected[i].link; REQUIRE(FileSystemManager::ResolveSymlinks(&path)); CHECK(FileSystemManager::Exists(path) == kSymExpected[i].resolvesToExisting); } } TEST_CASE_METHOD(SymlinkTestsFixture, "ReadDirectoryEntriesRecursive", "[file_system]") { size_t count{}; auto use_entry = [&count](std::filesystem::path const& /*name*/, bool /*is_tree*/) { ++count; return true; }; SECTION("Check directory is complete") { REQUIRE(FileSystemManager::ReadDirectoryEntriesRecursive(root_dir_, use_entry)); CHECK(count == num_entries_); } SECTION("Check directory with missing paths") { REQUIRE(FileSystemManager::ReadDirectoryEntriesRecursive( root_dir_, use_entry, /*ignored_subdirs=*/ {"baz"})); CHECK(count == num_root_file_entries_); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/git_repo.test.cpp000066400000000000000000000710651516554100600304210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_repo.hpp" #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/atomic.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hex_string.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kRootCommit = std::string{"3ecce3f5b19ad7941c6354d65d841590662f33ef"}; auto const kRootId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kBazId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"}; auto const kFooId = std::string{"19102815663d23f8b75a47e7a01965dcdc96468c"}; auto const kBarId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kFooBarTreeId = std::string{"27b32561185c2825150893774953906c6daa6798"}; } // namespace class TestUtils { public: [[nodiscard]] static auto GetTestDir() noexcept -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/other_tools"; } [[nodiscard]] static auto GetRepoPath() noexcept -> std::filesystem::path { return GetTestDir() / "test_git_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); } // The checkout will make the content available, as well as the HEAD ref [[nodiscard]] static auto CreateTestRepoWithCheckout( bool is_bare = false) noexcept -> std::optional { auto repo_path = CreateTestRepo(is_bare); REQUIRE(repo_path); auto cmd = fmt::format("git --git-dir={} --work-tree={} checkout master", QuoteForShell(is_bare ? repo_path->string() : (*repo_path / ".git").string()), QuoteForShell(repo_path->string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] static auto CreateTestRepo(bool is_bare = false) noexcept -> std::optional { auto repo_path = GetRepoPath(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } private: // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static inline std::atomic counter = 0; }; TEST_CASE("Open Git repo", "[git_repo]") { SECTION("Fake bare repository") { auto repo_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); // same odb, same GitCAS CHECK(repo->IsRepoFake()); } SECTION("Fake non-bare repository") { auto repo_path = TestUtils::CreateTestRepo(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); // same odb, same GitCAS CHECK(repo->IsRepoFake()); } SECTION("Real bare repository") { auto repo_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(*repo_path); REQUIRE(repo); CHECK_FALSE(repo->GetGitCAS() == cas); // same odb, different GitCAS CHECK_FALSE(repo->IsRepoFake()); } SECTION("Real non-bare repository") { auto repo_path = TestUtils::CreateTestRepo(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(*repo_path); REQUIRE(repo); CHECK_FALSE(repo->GetGitCAS() == cas); // same odb, different GitCAS CHECK_FALSE(repo->IsRepoFake()); } SECTION("Non-existing repository") { auto repo = GitRepo::Open("does_not_exist"); REQUIRE(repo == std::nullopt); } SECTION("Initialize and open bare repository") { auto repo_path = TestUtils::GetRepoPath(); auto repo = GitRepo::InitAndOpen(repo_path, /*is_bare=*/true); REQUIRE(repo); CHECK_FALSE(repo->IsRepoFake()); } SECTION("Real non-bare repository with checkout") { auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); CHECK(repo->IsRepoFake()); } } TEST_CASE("Single-threaded real repository local operations", "[git_repo]") { // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); SECTION("Commit directory") { // make blank repo auto repo_commit_path = TestUtils::GetRepoPath(); auto repo_commit = GitRepo::InitAndOpen(repo_commit_path, /*is_bare=*/false); REQUIRE(repo_commit); CHECK_FALSE(repo_commit->IsRepoFake()); // add blank file REQUIRE(FileSystemManager::WriteFile( "test no 1", repo_commit_path / "test1.txt", true)); REQUIRE(FileSystemManager::WriteFile( "test no 2", repo_commit_path / "test2.txt", true)); // commit subdir auto commit = repo_commit->CommitDirectory( repo_commit_path, "test commit", logger); CHECK(commit); } SECTION("Tag commit") { auto repo_tag_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_tag_path); auto repo_tag = GitRepo::Open(*repo_tag_path); REQUIRE(repo_tag); CHECK_FALSE(repo_tag->IsRepoFake()); CHECK(repo_tag->KeepTag(kRootCommit, "test tag", logger)); } SECTION("Get head commit") { auto repo_head_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_head_path); auto repo_head = GitRepo::Open(*repo_head_path); REQUIRE(repo_head); auto head_commit = repo_head->GetHeadCommit(logger); REQUIRE(head_commit); CHECK(*head_commit == kRootCommit); } SECTION("Fetch with base refspecs from path") { // make bare real repo to fetch into auto path_fetch_all = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(path_fetch_all); auto repo_fetch_all = GitRepo::Open(*path_fetch_all); // fetch all CHECK(repo_fetch_all->FetchFromPath( nullptr, *path_fetch_all, std::nullopt, logger)); } SECTION("Fetch branch from path") { // make bare real repo to fetch into auto path_fetch_branch = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(path_fetch_branch); auto repo_fetch_branch = GitRepo::Open(*path_fetch_branch); REQUIRE(repo_fetch_branch); // fetch branch CHECK(repo_fetch_branch->FetchFromPath( nullptr, *path_fetch_branch, "master", logger)); } SECTION("Tag tree") { auto repo_tag_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_tag_path); auto repo_tag = GitRepo::Open(*repo_tag_path); REQUIRE(repo_tag); CHECK_FALSE(repo_tag->IsRepoFake()); // tag tree already root of a commit CHECK(repo_tag->KeepTree(kRootId, "test tag 1", logger)); // tag tree part of another commit CHECK(repo_tag->KeepTree(kBazId, "test tag 2", logger)); // tag uncommitted tree auto foo_bar = GitRepo::tree_entries_t{ {FromHexString(kFooId).value_or({}), {GitRepo::TreeEntry{"foo", ObjectType::File}}}, {FromHexString(kBarId).value_or({}), {GitRepo::TreeEntry{"bar", ObjectType::Executable}}}}; auto foo_bar_id = repo_tag->CreateTree(foo_bar); REQUIRE(foo_bar_id); auto tree_id = ToHexString(*foo_bar_id); CHECK(tree_id == kFooBarTreeId); CHECK(repo_tag->KeepTree(tree_id, "test tag 3", logger)); } } // NOTE: "fake" repo ops tests split into two batches as workaround for // CATCH2_INTERNAL_TEST_16 function size/complexity threshold exceeding lint // warning TEST_CASE("Single-threaded fake repository operations -- batch 1", "[git_repo]") { auto const storage_config = TestStorageConfig::Create(); auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); REQUIRE(repo->GetGitCAS() == cas); REQUIRE(repo->IsRepoFake()); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); SECTION("Check tree exists") { auto res = repo->CheckTreeExists(kRootId, logger); REQUIRE(res); CHECK(*res); res = repo->CheckTreeExists(kBazId, logger); REQUIRE(res); CHECK(*res); res = repo->CheckTreeExists(kFooId, logger); REQUIRE(res); CHECK_FALSE(*res); } SECTION("Check blob exists") { auto res = repo->CheckBlobExists(kFooId, logger); REQUIRE(res); CHECK(*res); res = repo->CheckBlobExists(kBarId, logger); REQUIRE(res); CHECK(*res); res = repo->CheckBlobExists(kBazId, logger); REQUIRE(res); CHECK_FALSE(*res); } SECTION("Write and read blobs") { SECTION("Existing blobs") { auto res = repo->TryReadBlob(kFooId, logger); REQUIRE(res.first); REQUIRE(res.second); CHECK(res.second == "foo"); res = repo->TryReadBlob(kBarId, logger); REQUIRE(res.first); REQUIRE(res.second); CHECK(res.second == "bar"); res = repo->TryReadBlob(kBazId, logger); REQUIRE(res.first); // search succeeded... CHECK_FALSE(res.second); // ...but blob not found } SECTION("New blobs in existing repo") { auto w = repo->WriteBlob("foobar", logger); REQUIRE(w); auto blob_id = w.value(); auto r = repo->TryReadBlob(blob_id, logger); REQUIRE(r.first); REQUIRE(r.second); CHECK(r.second == "foobar"); } SECTION("Overwrite blob does not fail") { auto w = repo->WriteBlob("foo", logger); REQUIRE(w); CHECK(w == kFooId); } SECTION("New blobs in bare repo") { // make blank repo auto repo_path = TestUtils::GetRepoPath(); auto repo = GitRepo::InitAndOpen(repo_path, /*is_bare=*/false); REQUIRE(repo); CHECK_FALSE(repo->IsRepoFake()); auto w = repo->WriteBlob("foobar", logger); REQUIRE(w); auto blob_id = w.value(); auto r = repo->TryReadBlob(blob_id, logger); REQUIRE(r.first); REQUIRE(r.second); CHECK(r.second == "foobar"); } } SECTION("Check if commit exists in repository") { SECTION("Repository containing commit") { auto path_containing = TestUtils::CreateTestRepo(); REQUIRE(path_containing); auto cas_containing = GitCAS::Open(*path_containing); REQUIRE(cas_containing); auto repo_containing = GitRepo::Open(cas_containing); REQUIRE(repo_containing); auto result_containing = repo_containing->CheckCommitExists(kRootCommit, logger); CHECK(*result_containing); } SECTION("Repository not containing commit") { auto path_non_bare = TestUtils::GetRepoPath(); { auto repo_tmp = GitRepo::InitAndOpen(path_non_bare, /*is_bare=*/false); REQUIRE(repo_tmp); } auto cas_non_bare = GitCAS::Open(path_non_bare); REQUIRE(cas_non_bare); auto repo_non_bare = GitRepo::Open(cas_non_bare); REQUIRE(repo_non_bare); auto result_non_bare = repo_non_bare->CheckCommitExists(kRootCommit, logger); CHECK_FALSE(*result_non_bare); } } SECTION("Fetch from local repository via temporary repository") { SECTION("Fetch all") { // set repo to fetch into auto path_fetch_all = TestUtils::GetRepoPath(); auto repo_fetch_all = GitRepo::InitAndOpen(path_fetch_all, /*is_bare=*/true); REQUIRE(repo_fetch_all); // check commit is not there before fetch CHECK_FALSE( *repo_fetch_all->CheckCommitExists(kRootCommit, logger)); // fetch all with base refspecs REQUIRE(repo_fetch_all->LocalFetchViaTmpRepo( storage_config.Get(), *repo_path, std::nullopt, logger)); // check commit is there after fetch CHECK(*repo_fetch_all->CheckCommitExists(kRootCommit, logger)); } SECTION("Fetch branch") { // set repo to fetch into auto path_fetch_branch = TestUtils::GetRepoPath(); auto repo_fetch_branch = GitRepo::InitAndOpen(path_fetch_branch, /*is_bare=*/true); REQUIRE(repo_fetch_branch); // check commit is not there before fetch CHECK_FALSE( *repo_fetch_branch->CheckCommitExists(kRootCommit, logger)); // fetch branch REQUIRE(repo_fetch_branch->LocalFetchViaTmpRepo( storage_config.Get(), *repo_path, "master", logger)); // check commit is there after fetch CHECK(*repo_fetch_branch->CheckCommitExists(kRootCommit, logger)); } } } TEST_CASE("Single-threaded fake repository operations -- batch 2", "[git_repo]") { auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); REQUIRE(repo->GetGitCAS() == cas); REQUIRE(repo->IsRepoFake()); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); SECTION("Get subtree entry id from commit") { SECTION("Get base tree id") { auto entry_root_c = repo->GetSubtreeFromCommit(kRootCommit, ".", logger); REQUIRE(entry_root_c); CHECK(*entry_root_c == kRootId); } SECTION("Get inner tree id") { auto entry_baz_c = repo->GetSubtreeFromCommit(kRootCommit, "baz", logger); REQUIRE(entry_baz_c); CHECK(*entry_baz_c == kBazId); } } SECTION("Get subtree entry id from root tree id") { SECTION("Get base tree id") { auto entry_root_t = repo->GetSubtreeFromTree(kRootId, ".", logger); REQUIRE(entry_root_t); CHECK(*entry_root_t == kRootId); } SECTION("Get inner tree id") { auto entry_baz_t = repo->GetSubtreeFromTree(kRootId, "baz", logger); REQUIRE(entry_baz_t); CHECK(*entry_baz_t == kBazId); } } SECTION("Find repository root from path") { SECTION("Non-bare repository") { auto root_path = GitRepo::GetRepoRootFromPath(*repo_path, logger); REQUIRE(root_path); CHECK(*root_path == *repo_path); auto root_path_from_baz = GitRepo::GetRepoRootFromPath(*repo_path / "baz", logger); REQUIRE(root_path_from_baz); CHECK(*root_path_from_baz == *repo_path); auto root_path_from_bazfoo = GitRepo::GetRepoRootFromPath(*repo_path / "baz/foo", logger); REQUIRE(root_path_from_bazfoo); CHECK(*root_path_from_bazfoo == *repo_path); auto root_path_non_exist = GitRepo::GetRepoRootFromPath("does_not_exist", logger); REQUIRE(root_path_non_exist); CHECK(root_path_non_exist->empty()); } SECTION("Bare repository") { auto bare_repo_path = TestUtils::CreateTestRepo(true); REQUIRE(bare_repo_path); auto bare_cas = GitCAS::Open(*bare_repo_path); REQUIRE(bare_cas); auto bare_repo = GitRepo::Open(bare_cas); REQUIRE(bare_repo); auto bare_repo_root_path = GitRepo::GetRepoRootFromPath(*bare_repo_path, logger); REQUIRE(bare_repo_root_path); CHECK(*bare_repo_root_path == *bare_repo_path); } } SECTION("Get subtree entry id from path") { SECTION("Get base tree id") { auto entry_root_p = repo->GetSubtreeFromPath(*repo_path, kRootCommit, logger); REQUIRE(entry_root_p); CHECK(*entry_root_p == kRootId); } SECTION("Get inner tree id") { auto path_baz = *repo_path / "baz"; auto entry_baz_p = repo->GetSubtreeFromPath(path_baz, kRootCommit, logger); REQUIRE(entry_baz_p); CHECK(*entry_baz_p == kBazId); } } SECTION("Read object from tree by relative path") { SECTION("Non-existing") { auto obj_info = repo->GetObjectByPathFromTree(kRootId, "does_not_exist"); CHECK_FALSE(obj_info); } SECTION("File") { auto obj_info = repo->GetObjectByPathFromTree(kRootId, "foo"); REQUIRE(obj_info); CHECK(obj_info->id == kFooId); CHECK(obj_info->type == ObjectType::File); CHECK_FALSE(obj_info->symlink_content); } SECTION("Tree") { auto obj_info = repo->GetObjectByPathFromTree(kRootId, "baz"); REQUIRE(obj_info); CHECK(obj_info->id == kBazId); CHECK(obj_info->type == ObjectType::Tree); CHECK_FALSE(obj_info->symlink_content); } SECTION("Symlink") { auto obj_info = repo->GetObjectByPathFromTree(kRootId, "baz/bar_l"); REQUIRE(obj_info); CHECK(obj_info->id == kBarId); CHECK(obj_info->type == ObjectType::Symlink); CHECK(obj_info->symlink_content); CHECK(*obj_info->symlink_content == "bar"); } } } TEST_CASE("Multi-threaded fake repository operations", "[git_repo]") { auto const storage_config = TestStorageConfig::Create(); /* Test all fake repository operations while being done in parallel. They are supposed to be thread-safe, so no conflicts should exist. */ // define remote, for ops that need it auto remote_repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(remote_repo_path); auto remote_cas = GitCAS::Open(*remote_repo_path); REQUIRE(remote_cas); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); // setup threading constexpr auto kNumThreads = 100; atomic starting_signal{false}; std::vector threads{}; threads.reserve(kNumThreads); SECTION("Lookups in the same ODB") { constexpr int kNumCases = 10; for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&storage_config, &remote_cas, &remote_repo_path, &logger, &starting_signal](int tid) { starting_signal.wait(false); // cases based on thread number switch (tid % kNumCases) { case 0: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Get subtree entry id from commit auto entry_baz_c = remote_repo->GetSubtreeFromCommit( kRootCommit, "baz", logger); REQUIRE(entry_baz_c); CHECK(*entry_baz_c == kBazId); } break; case 1: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Get subtree entry id from root tree id auto entry_baz_t = remote_repo->GetSubtreeFromTree( kRootId, "baz", logger); REQUIRE(entry_baz_t); CHECK(*entry_baz_t == kBazId); } break; case 2: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Find repository root from path auto root_path_from_bazbar = GitRepo::GetRepoRootFromPath( *remote_repo_path / "baz/bar", logger); REQUIRE(root_path_from_bazbar); CHECK(*root_path_from_bazbar == *remote_repo_path); } break; case 3: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Lookup trees auto res = remote_repo->CheckTreeExists(kRootId, logger); REQUIRE(res); CHECK(*res); res = remote_repo->CheckTreeExists(kBazId, logger); REQUIRE(res); CHECK(*res); } break; case 4: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Lookup blobs auto res = remote_repo->CheckBlobExists(kFooId, logger); REQUIRE(res); CHECK(*res); res = remote_repo->CheckBlobExists(kBarId, logger); REQUIRE(res); CHECK(*res); } break; // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) case 5: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Read blobs auto res = remote_repo->TryReadBlob(kFooId, logger); REQUIRE(res.first); REQUIRE(res.second); CHECK(res.second == "foo"); res = remote_repo->TryReadBlob(kBarId, logger); REQUIRE(res.first); REQUIRE(res.second); CHECK(res.second == "bar"); } break; // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) case 6: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Write blobs auto res = remote_repo->WriteBlob( std::to_string(tid), logger); REQUIRE(res); // ...including existing content res = remote_repo->WriteBlob("foo", logger); REQUIRE(res); CHECK(res == kFooId); } break; // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) case 7: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // Get subtree entry id from path auto path_baz = *remote_repo_path / "baz"; auto entry_baz_p = remote_repo->GetSubtreeFromPath( path_baz, kRootCommit, logger); REQUIRE(entry_baz_p); CHECK(*entry_baz_p == kBazId); } break; // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) case 8: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); auto result_containing = remote_repo->CheckCommitExists(kRootCommit, logger); CHECK(*result_containing); } break; // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) case 9: { auto remote_repo = GitRepo::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->IsRepoFake()); // fetch all REQUIRE(remote_repo->LocalFetchViaTmpRepo( storage_config.Get(), *remote_repo_path, std::nullopt, logger)); } break; default: REQUIRE(false); } }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/git_tree.test.cpp000066400000000000000000001101741516554100600304060ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/git_tree.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "fmt/core.h" #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/atomic.hpp" #include "src/utils/cpp/hex_string.hpp" #include "test/utils/container_matchers.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo.bundle"}; auto const kTreeId = std::string{"c610db170fbcad5f2d66fe19972495923f3b2536"}; auto const kFooId = std::string{"19102815663d23f8b75a47e7a01965dcdc96468c"}; auto const kBarId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kFailId = std::string{"0123456789abcdef0123456789abcdef01234567"}; auto const kBundleSymPath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kTreeSymId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kBazLinkId = std::string{"3f9538666251333f5fa519e01eb267d371ca9c78"}; auto const kBazBarLinkId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kFooLinkId = std::string{"b24736f10d3c60015386047ebc98b4ab63056041"}; [[nodiscard]] auto HexToRaw(std::string const& hex) -> std::string { return FromHexString(hex).value_or({}); } [[nodiscard]] auto RawToHex(std::string const& raw) -> std::string { return ToHexString(raw); } [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/file_system"; } [[nodiscard]] auto CreateTestRepo(bool is_bare = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] auto CreateTestRepoSymlinks(bool is_bare = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo_symlinks" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundleSymPath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } class SymlinksChecker final { public: explicit SymlinksChecker(gsl::not_null const& cas) noexcept : cas_{*cas} {} [[nodiscard]] auto operator()( std::vector const& ids) const noexcept -> bool { return std::all_of( ids.begin(), ids.end(), [&cas = cas_](ArtifactDigest const& id) { return cas .ReadObject(id.hash(), /*is_hex_id=*/true, /*as_valid_symlink=*/true) .has_value(); }); }; private: GitCAS const& cas_; }; } // namespace TEST_CASE("Open Git CAS", "[git_cas]") { SECTION("Bare repository") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); CHECK(GitCAS::Open(*repo_path)); } SECTION("Non-bare repository") { auto repo_path = CreateTestRepo(false); REQUIRE(repo_path); CHECK(GitCAS::Open(*repo_path)); } SECTION("Non-existing repository") { CHECK_FALSE(GitCAS::Open("does_not_exist")); } } TEST_CASE("Read Git Objects", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); SECTION("valid ids") { CHECK(cas->ReadObject(kFooId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kFooId), /*is_hex_id=*/false)); CHECK(cas->ReadObject(kBarId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kBarId), /*is_hex_id=*/false)); CHECK(cas->ReadObject(kTreeSymId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kTreeSymId), /*is_hex_id=*/false)); CHECK(cas->ReadObject(kBazBarLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kBazBarLinkId), /*is_hex_id=*/false)); CHECK(cas->ReadObject(kBazBarLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kBazBarLinkId), /*is_hex_id=*/false)); CHECK(cas->ReadObject(kFooLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadObject(HexToRaw(kFooLinkId), /*is_hex_id=*/false)); } SECTION("invalid ids") { CHECK_FALSE(cas->ReadObject("", /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadObject("", /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadObject(kFailId, /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadObject(HexToRaw(kFailId), /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadObject(RawToHex("to_short"), /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadObject("to_short", /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadObject("invalid_chars", /*is_hex_id=*/true)); } } TEST_CASE("Read Git Headers", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); SECTION("valid ids") { CHECK(cas->ReadHeader(kFooId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kFooId), /*is_hex_id=*/false)); CHECK(cas->ReadHeader(kBarId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kBarId), /*is_hex_id=*/false)); CHECK(cas->ReadHeader(kTreeSymId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kTreeSymId), /*is_hex_id=*/false)); CHECK(cas->ReadHeader(kBazBarLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kBazBarLinkId), /*is_hex_id=*/false)); CHECK(cas->ReadHeader(kBazBarLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kBazBarLinkId), /*is_hex_id=*/false)); CHECK(cas->ReadHeader(kFooLinkId, /*is_hex_id=*/true)); CHECK(cas->ReadHeader(HexToRaw(kFooLinkId), /*is_hex_id=*/false)); } SECTION("invalid ids") { CHECK_FALSE(cas->ReadHeader("", /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadHeader("", /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadHeader(kFailId, /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadHeader(HexToRaw(kFailId), /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadHeader(RawToHex("to_short"), /*is_hex_id=*/true)); CHECK_FALSE(cas->ReadHeader("to_short", /*is_hex_id=*/false)); CHECK_FALSE(cas->ReadHeader("invalid_chars", /*is_hex_id=*/true)); } } TEST_CASE("Read Git Trees", "[git_cas]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("invalid trees") { CHECK_FALSE(repo->ReadTree("", check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree("", check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE( repo->ReadTree(kFailId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kFailId), check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree( RawToHex("to_short"), check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE( repo->ReadTree("to_short", check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree( "invalid_chars", check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree(kFooId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kFooId), check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree(kBarId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kBarId), check_symlinks, /*is_hex_id=*/false)); } SECTION("valid trees") { auto entries0 = repo->ReadTree(kTreeId, check_symlinks, /*is_hex_id=*/true); auto entries1 = repo->ReadTree( HexToRaw(kTreeId), check_symlinks, /*is_hex_id=*/false); REQUIRE(entries0); REQUIRE(entries1); CHECK(*entries0 == *entries1); } } TEST_CASE("Read Git Trees with symlinks -- ignore special", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("invalid trees") { CHECK_FALSE(repo->ReadTree( "", check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree( "", check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(kFailId, check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(HexToRaw(kFailId), check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(RawToHex("to_short"), check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree("to_short", check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree("invalid_chars", check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(kFooId, check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(HexToRaw(kFooId), check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(kBarId, check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true)); CHECK_FALSE(repo->ReadTree(HexToRaw(kBarId), check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true)); } SECTION("valid trees") { auto entries0 = repo->ReadTree(kTreeSymId, check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true); auto entries1 = repo->ReadTree(HexToRaw(kTreeSymId), check_symlinks, /*is_hex_id=*/false, /*ignore_special=*/true); REQUIRE(entries0); REQUIRE(entries1); CHECK(*entries0 == *entries1); } } TEST_CASE("Read Git Trees with symlinks -- allow non-upwards", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("invalid trees") { CHECK_FALSE(repo->ReadTree("", check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree("", check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE( repo->ReadTree(kFailId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kFailId), check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree( RawToHex("to_short"), check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE( repo->ReadTree("to_short", check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree( "invalid_chars", check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree(kFooId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kFooId), check_symlinks, /*is_hex_id=*/false)); CHECK_FALSE(repo->ReadTree(kBarId, check_symlinks, /*is_hex_id=*/true)); CHECK_FALSE(repo->ReadTree( HexToRaw(kBarId), check_symlinks, /*is_hex_id=*/false)); } SECTION("valid trees") { auto entries0 = repo->ReadTree(kTreeSymId, check_symlinks, /*is_hex_id=*/true); auto entries1 = repo->ReadTree( HexToRaw(kTreeSymId), check_symlinks, /*is_hex_id=*/false); REQUIRE(entries0); REQUIRE(entries1); CHECK(*entries0 == *entries1); } } TEST_CASE("Create Git Trees", "[git_cas]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("empty tree") { auto tree_id = repo->CreateTree({}); REQUIRE(tree_id); CHECK(ToHexString(*tree_id) == "4b825dc642cb6eb9a060e54bf8d69288fbee4904"); } SECTION("existing tree") { auto entries = repo->ReadTree(kTreeId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); auto tree_id = repo->CreateTree(*entries); REQUIRE(tree_id); CHECK(ToHexString(*tree_id) == kTreeId); } SECTION("entry order") { auto foo_bar = GitRepo::tree_entries_t{ {HexToRaw(kFooId), {GitRepo::TreeEntry{"foo", ObjectType::File}, GitRepo::TreeEntry{"bar", ObjectType::Executable}}}}; auto foo_bar_id = repo->CreateTree(foo_bar); REQUIRE(foo_bar_id); auto bar_foo = GitRepo::tree_entries_t{ {HexToRaw(kFooId), {GitRepo::TreeEntry{"bar", ObjectType::Executable}, GitRepo::TreeEntry{"foo", ObjectType::File}}}}; auto bar_foo_id = repo->CreateTree(bar_foo); REQUIRE(bar_foo_id); CHECK(foo_bar_id == bar_foo_id); } } TEST_CASE("Create Git Trees with symlinks", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("existing tree with symlinks -- ignore special") { auto entries = repo->ReadTree(kTreeSymId, check_symlinks, /*is_hex_id=*/true, /*ignore_special=*/true); REQUIRE(entries); auto tree_id = repo->CreateTree(*entries); REQUIRE(tree_id); // if at least one symlink exists, it gets ignored and the tree id will // not match as it is NOT recomputed! CHECK_FALSE(ToHexString(*tree_id) == kTreeSymId); } SECTION("existing tree with symlinks -- allow non-upwards") { auto entries = repo->ReadTree(kTreeSymId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); auto tree_id = repo->CreateTree(*entries); REQUIRE(tree_id); // all the symlinks in the test repo are non-upwards, so the tree should // be recreated exactly and id should thus match CHECK(ToHexString(*tree_id) == kTreeSymId); } } TEST_CASE("Read Git Tree Data", "[git_cas]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("empty tree") { auto entries = GitRepo::ReadTreeData("", "4b825dc642cb6eb9a060e54bf8d69288fbee4904", check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); CHECK(entries->empty()); } SECTION("existing tree") { auto entries = repo->ReadTree(kTreeId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); auto data = cas->ReadObject(kTreeId, /*is_hex_id=*/true); REQUIRE(data); auto from_data = GitRepo::ReadTreeData( *data, kTreeId, check_symlinks, /*is_hex_id=*/true); REQUIRE(from_data); CHECK(*from_data == *entries); } } TEST_CASE("Read Git Tree Data with non-upwards symlinks", "[git_cas]") { auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("empty tree") { auto entries = GitRepo::ReadTreeData("", "4b825dc642cb6eb9a060e54bf8d69288fbee4904", check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); CHECK(entries->empty()); } SECTION("existing tree") { auto entries = repo->ReadTree(kTreeSymId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); auto data = cas->ReadObject(kTreeSymId, /*is_hex_id=*/true); REQUIRE(data); auto from_data = GitRepo::ReadTreeData( *data, kTreeSymId, check_symlinks, /*is_hex_id=*/true); REQUIRE(from_data); CHECK(*from_data == *entries); } } TEST_CASE("Create Shallow Git Trees", "[git_cas]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepo::Open(cas); REQUIRE(repo); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; SECTION("empty tree") { auto tree = GitRepo::CreateShallowTree({}); REQUIRE(tree); CHECK(ToHexString(tree->first) == "4b825dc642cb6eb9a060e54bf8d69288fbee4904"); CHECK(tree->second.empty()); } SECTION("existing tree from other CAS") { auto entries = repo->ReadTree(kTreeId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); auto tree = GitRepo::CreateShallowTree(*entries); REQUIRE(tree); CHECK(ToHexString(tree->first) == kTreeId); CHECK_FALSE(tree->second.empty()); } } TEST_CASE("Read Git Tree", "[git_tree]") { SECTION("Bare repository") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); CHECK(GitTree::Read(*repo_path, kTreeId)); CHECK_FALSE(GitTree::Read(*repo_path, "wrong_tree_id")); } SECTION("Non-bare repository") { auto repo_path = CreateTestRepo(false); REQUIRE(repo_path); CHECK(GitTree::Read(*repo_path, kTreeId)); CHECK_FALSE(GitTree::Read(*repo_path, "wrong_tree_id")); } } TEST_CASE("Read Git Tree with non-upwards symlinks", "[git_tree]") { auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); CHECK(GitTree::Read(*repo_path, kTreeSymId)); CHECK_FALSE(GitTree::Read(*repo_path, "wrong_tree_id")); } TEST_CASE("Lookup entries by name", "[git_tree]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeId); REQUIRE(tree_root); auto entry_foo = tree_root->LookupEntryByName("foo"); REQUIRE(entry_foo); CHECK(entry_foo->IsBlob()); CHECK(entry_foo->Type() == ObjectType::File); auto blob_foo = entry_foo->Blob(); REQUIRE(blob_foo); CHECK(*blob_foo == "foo"); CHECK(blob_foo->size() == 3); CHECK(blob_foo->size() == *entry_foo->Size()); auto entry_bar = tree_root->LookupEntryByName("bar"); REQUIRE(entry_bar); CHECK(entry_bar->IsBlob()); CHECK(entry_bar->Type() == ObjectType::Executable); auto blob_bar = entry_bar->Blob(); REQUIRE(blob_bar); CHECK(*blob_bar == "bar"); CHECK(blob_bar->size() == 3); CHECK(blob_bar->size() == *entry_bar->Size()); auto entry_baz = tree_root->LookupEntryByName("baz"); REQUIRE(entry_baz); CHECK(entry_baz->IsTree()); CHECK(entry_baz->Type() == ObjectType::Tree); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByName("fool")); CHECK_FALSE(tree_root->LookupEntryByName("barn")); CHECK_FALSE(tree_root->LookupEntryByName("bazel")); } SECTION("Lookup entries in sub-tree") { auto const& tree_baz = entry_baz->Tree(); REQUIRE(tree_baz); auto entry_baz_foo = tree_baz->LookupEntryByName("foo"); REQUIRE(entry_baz_foo); CHECK(entry_baz_foo->IsBlob()); CHECK(entry_baz_foo->Hash() == entry_foo->Hash()); auto entry_baz_bar = tree_baz->LookupEntryByName("bar"); REQUIRE(entry_baz_bar); CHECK(entry_baz_bar->IsBlob()); CHECK(entry_baz_bar->Hash() == entry_bar->Hash()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_baz->LookupEntryByName("fool")); CHECK_FALSE(tree_baz->LookupEntryByName("barn")); CHECK_FALSE(tree_baz->LookupEntryByName("bazel")); } } } TEST_CASE("Lookup symlinks by name", "[git_tree]") { auto repo_path = CreateTestRepoSymlinks(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeSymId); REQUIRE(tree_root); auto entry_foo_l = tree_root->LookupEntryByName("foo_l"); REQUIRE(entry_foo_l); CHECK(entry_foo_l->IsBlob()); CHECK(entry_foo_l->Type() == ObjectType::Symlink); auto blob_foo_l = entry_foo_l->Blob(); REQUIRE(blob_foo_l); CHECK(*blob_foo_l == "baz/foo"); CHECK(blob_foo_l->size() == 7); CHECK(blob_foo_l->size() == *entry_foo_l->Size()); auto entry_baz_l = tree_root->LookupEntryByName("baz_l"); REQUIRE(entry_baz_l); CHECK(entry_baz_l->IsBlob()); CHECK(entry_baz_l->Type() == ObjectType::Symlink); auto blob_baz_l = entry_baz_l->Blob(); REQUIRE(blob_baz_l); CHECK(*blob_baz_l == "baz"); CHECK(blob_baz_l->size() == 3); CHECK(blob_baz_l->size() == *entry_baz_l->Size()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByName("fool")); CHECK_FALSE(tree_root->LookupEntryByName("barn")); CHECK_FALSE(tree_root->LookupEntryByName("bazel")); } SECTION("Lookup symlinks in sub-tree") { auto entry_baz = tree_root->LookupEntryByName("baz"); REQUIRE(entry_baz); CHECK(entry_baz->IsTree()); CHECK(entry_baz->Type() == ObjectType::Tree); auto const& tree_baz = entry_baz->Tree(); REQUIRE(tree_baz); auto entry_baz_bar = tree_baz->LookupEntryByName("bar"); REQUIRE(entry_baz_bar); CHECK(entry_baz_bar->IsBlob()); CHECK(entry_baz_bar->Type() == ObjectType::Executable); auto entry_baz_bar_l = tree_baz->LookupEntryByName("bar_l"); REQUIRE(entry_baz_bar_l); CHECK(entry_baz_bar_l->IsBlob()); CHECK(entry_baz_bar_l->Type() == ObjectType::Symlink); // the hash of the symlink content should be the same as the file CHECK(entry_baz_bar_l->Hash() == entry_baz_bar->Hash()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_baz->LookupEntryByName("fool")); CHECK_FALSE(tree_baz->LookupEntryByName("barn")); CHECK_FALSE(tree_baz->LookupEntryByName("bazel")); } } } TEST_CASE("Lookup entries by path", "[git_tree]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeId); REQUIRE(tree_root); auto entry_foo = tree_root->LookupEntryByPath("foo"); REQUIRE(entry_foo); CHECK(entry_foo->IsBlob()); CHECK(entry_foo->Type() == ObjectType::File); auto blob_foo = entry_foo->Blob(); REQUIRE(blob_foo); CHECK(*blob_foo == "foo"); CHECK(blob_foo->size() == 3); CHECK(blob_foo->size() == *entry_foo->Size()); auto entry_bar = tree_root->LookupEntryByPath("bar"); REQUIRE(entry_bar); CHECK(entry_bar->IsBlob()); CHECK(entry_bar->Type() == ObjectType::Executable); auto blob_bar = entry_bar->Blob(); REQUIRE(blob_bar); CHECK(*blob_bar == "bar"); CHECK(blob_bar->size() == 3); CHECK(blob_bar->size() == *entry_bar->Size()); auto entry_baz = tree_root->LookupEntryByPath("baz"); REQUIRE(entry_baz); CHECK(entry_baz->IsTree()); CHECK(entry_baz->Type() == ObjectType::Tree); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByPath("fool")); CHECK_FALSE(tree_root->LookupEntryByPath("barn")); CHECK_FALSE(tree_root->LookupEntryByPath("bazel")); } SECTION("Lookup entries in sub-tree") { auto entry_baz_foo = tree_root->LookupEntryByPath("baz/foo"); REQUIRE(entry_baz_foo); CHECK(entry_baz_foo->IsBlob()); CHECK(entry_baz_foo->Hash() == entry_foo->Hash()); auto entry_baz_bar = tree_root->LookupEntryByPath("baz/bar"); REQUIRE(entry_baz_bar); CHECK(entry_baz_bar->IsBlob()); CHECK(entry_baz_bar->Hash() == entry_bar->Hash()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByPath("baz/fool")); CHECK_FALSE(tree_root->LookupEntryByPath("baz/barn")); CHECK_FALSE(tree_root->LookupEntryByPath("baz/bazel")); } } } TEST_CASE("Lookup symlinks by path", "[git_tree]") { auto repo_path = CreateTestRepoSymlinks(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeSymId); REQUIRE(tree_root); auto entry_foo_l = tree_root->LookupEntryByPath("foo_l"); REQUIRE(entry_foo_l); CHECK(entry_foo_l->IsBlob()); CHECK(entry_foo_l->Type() == ObjectType::Symlink); auto blob_foo_l = entry_foo_l->Blob(); REQUIRE(blob_foo_l); CHECK(*blob_foo_l == "baz/foo"); CHECK(blob_foo_l->size() == 7); CHECK(blob_foo_l->size() == *entry_foo_l->Size()); auto entry_baz_l = tree_root->LookupEntryByPath("baz_l"); REQUIRE(entry_baz_l); CHECK(entry_baz_l->IsBlob()); CHECK(entry_baz_l->Type() == ObjectType::Symlink); auto blob_baz_l = entry_baz_l->Blob(); REQUIRE(blob_baz_l); CHECK(*blob_baz_l == "baz"); CHECK(blob_baz_l->size() == 3); CHECK(blob_baz_l->size() == *entry_baz_l->Size()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByPath("fool")); CHECK_FALSE(tree_root->LookupEntryByPath("barn")); CHECK_FALSE(tree_root->LookupEntryByPath("bazel")); } SECTION("Lookup symlinks in sub-tree") { auto entry_baz_bar = tree_root->LookupEntryByPath("baz/bar"); REQUIRE(entry_baz_bar); CHECK(entry_baz_bar->IsBlob()); CHECK(entry_baz_bar->Type() == ObjectType::Executable); auto entry_baz_bar_l = tree_root->LookupEntryByPath("baz/bar_l"); REQUIRE(entry_baz_bar_l); CHECK(entry_baz_bar_l->IsBlob()); CHECK(entry_baz_bar_l->Type() == ObjectType::Symlink); // the hash of the symlink content should be the same as the file CHECK(entry_baz_bar_l->Hash() == entry_baz_bar->Hash()); SECTION("Lookup missing entries") { CHECK_FALSE(tree_root->LookupEntryByPath("baz/fool")); CHECK_FALSE(tree_root->LookupEntryByPath("baz/barn")); CHECK_FALSE(tree_root->LookupEntryByPath("baz/bazel")); } } } TEST_CASE("Lookup entries by special names", "[git_tree]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeId); REQUIRE(tree_root); CHECK_FALSE(tree_root->LookupEntryByName(".")); // forbidden CHECK_FALSE(tree_root->LookupEntryByName("..")); // forbidden CHECK_FALSE(tree_root->LookupEntryByName("baz/")); // invalid name CHECK_FALSE(tree_root->LookupEntryByName("baz/foo")); // invalid name } TEST_CASE("Lookup entries by special paths", "[git_tree]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeId); REQUIRE(tree_root); SECTION("valid paths") { CHECK(tree_root->LookupEntryByPath("baz/")); CHECK(tree_root->LookupEntryByPath("baz/foo")); CHECK(tree_root->LookupEntryByPath("baz/../baz/")); CHECK(tree_root->LookupEntryByPath("./baz/")); CHECK(tree_root->LookupEntryByPath("./baz/foo")); CHECK(tree_root->LookupEntryByPath("./baz/../foo")); } SECTION("invalid paths") { CHECK_FALSE(tree_root->LookupEntryByPath(".")); // forbidden CHECK_FALSE(tree_root->LookupEntryByPath("..")); // outside of tree CHECK_FALSE(tree_root->LookupEntryByPath("/baz")); // outside of tree CHECK_FALSE(tree_root->LookupEntryByPath("baz/..")); // == '.' } } TEST_CASE("Iterate tree entries", "[git_tree]") { auto repo_path = CreateTestRepo(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeId); REQUIRE(tree_root); std::vector names{}; for (auto const& [name, entry] : *tree_root) { names.emplace_back(name); } CHECK_THAT(names, HasSameUniqueElementsAs>( {"foo", "bar", "baz"})); } TEST_CASE("Iterate tree entries with non-upwards symlinks", "[git_tree]") { auto repo_path = CreateTestRepoSymlinks(true); REQUIRE(repo_path); auto tree_root = GitTree::Read(*repo_path, kTreeSymId); REQUIRE(tree_root); std::vector names{}; for (auto const& [name, entry] : *tree_root) { names.emplace_back(name); } CHECK_THAT(names, HasSameUniqueElementsAs>( {"foo", "bar", "baz", "foo_l", "baz_l"})); } TEST_CASE("Thread-safety", "[git_tree]") { constexpr auto kNumThreads = 100; atomic starting_signal{false}; std::vector threads{}; threads.reserve(kNumThreads); auto repo_path = CreateTestRepoSymlinks(false); // checkout needed REQUIRE(repo_path); SECTION("Opening and reading from the same CAS") { for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&repo_path, &starting_signal](int tid) { starting_signal.wait(false); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); // every second thread reads bar instead of foo auto id = tid % 2 == 0 ? kFooId : kBarId; CHECK(cas->ReadObject(id, /*is_hex_id=*/true)); auto header = cas->ReadHeader(id, /*is_hex_id=*/true); CHECK(header->first == 3); CHECK(header->second == ObjectType::File); }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } SECTION("Parsing same tree with same CAS") { auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); // create symlinks checker auto const check_symlinks = SymlinksChecker{cas}; for (int id{}; id < kNumThreads; ++id) { threads.emplace_back([&cas, &starting_signal, check_symlinks]() { starting_signal.wait(false); auto repo = GitRepo::Open(cas); REQUIRE(repo); auto entries = repo->ReadTree( kTreeSymId, check_symlinks, /*is_hex_id=*/true); REQUIRE(entries); }); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } SECTION("Reading from different trees with same CAS") { for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&repo_path, &starting_signal](int tid) { starting_signal.wait(false); auto tree_root = GitTree::Read(*repo_path, kTreeSymId); REQUIRE(tree_root); auto entry_subdir = tree_root->LookupEntryByName("baz"); REQUIRE(entry_subdir); REQUIRE(entry_subdir->IsTree()); // every second thread reads subdir instead of root auto const& tree_read = tid % 2 == 0 ? tree_root : entry_subdir->Tree(); auto entry_foo = tree_read->LookupEntryByName("foo"); auto entry_bar = tree_read->LookupEntryByName("bar"); REQUIRE(entry_foo); REQUIRE(entry_bar); CHECK(entry_foo->Blob() == "foo"); CHECK(entry_bar->Blob() == "bar"); }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } SECTION("Reading from the same tree") { auto tree_root = GitTree::Read(*repo_path, kTreeSymId); REQUIRE(tree_root); for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&tree_root, &starting_signal](int tid) { // every second thread reads bar instead of foo auto name = tid % 2 == 0 ? std::string{"foo"} : std::string{"bar"}; starting_signal.wait(false); auto entry = tree_root->LookupEntryByName(name); REQUIRE(entry); CHECK(entry->Blob() == name); }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/object_cas.test.cpp000066400000000000000000000106601516554100600306770ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/object_cas.hpp" #include #include // has_value() #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" TEST_CASE("ObjectCAS", "[file_system]") { auto const storage_config = TestStorageConfig::Create(); auto gen_config = storage_config.Get().CreateGenerationConfig(0); std::string test_content{"test"}; auto test_digest = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, test_content); SECTION("CAS for files") { ObjectCAS cas{storage_config.Get().hash_function, gen_config.cas_f}; CHECK(not cas.BlobPath(test_digest)); SECTION("Add blob from bytes and verify") { // add blob auto cas_digest = cas.StoreBlobFromBytes(test_content); CHECK(cas_digest); CHECK(*cas_digest == test_digest); // verify blob auto blob_path = cas.BlobPath(*cas_digest); REQUIRE(blob_path); auto const cas_content = FileSystemManager::ReadFile(*blob_path); CHECK(cas_content.has_value()); CHECK(cas_content == test_content); CHECK(not FileSystemManager::IsExecutable(*blob_path)); } SECTION("Add blob from file") { CHECK(FileSystemManager::CreateDirectory("tmp")); CHECK(FileSystemManager::WriteFile(test_content, "tmp/test")); // add blob auto cas_digest = cas.StoreBlobFromFile("tmp/test"); CHECK(cas_digest); CHECK(*cas_digest == test_digest); // verify blob auto blob_path = cas.BlobPath(*cas_digest); REQUIRE(blob_path); auto const cas_content = FileSystemManager::ReadFile(*blob_path); CHECK(cas_content.has_value()); CHECK(cas_content == test_content); CHECK(not FileSystemManager::IsExecutable(*blob_path)); } } SECTION("CAS for executables") { ObjectCAS cas{ storage_config.Get().hash_function, gen_config.cas_x}; CHECK(not cas.BlobPath(test_digest)); SECTION("Add blob from bytes and verify") { // add blob auto cas_digest = cas.StoreBlobFromBytes(test_content); CHECK(cas_digest); CHECK(*cas_digest == test_digest); // verify blob auto blob_path = cas.BlobPath(*cas_digest); REQUIRE(blob_path); auto const cas_content = FileSystemManager::ReadFile(*blob_path); CHECK(cas_content.has_value()); CHECK(cas_content == test_content); CHECK(FileSystemManager::IsExecutable(*blob_path)); } SECTION("Add blob from file") { CHECK(FileSystemManager::CreateDirectory("tmp")); CHECK(FileSystemManager::WriteFile(test_content, "tmp/test")); // add blob auto cas_digest = cas.StoreBlobFromFile("tmp/test"); CHECK(cas_digest); CHECK(*cas_digest == test_digest); // verify blob auto blob_path = cas.BlobPath(*cas_digest); REQUIRE(blob_path); auto const cas_content = FileSystemManager::ReadFile(*blob_path); CHECK(cas_content.has_value()); CHECK(cas_content == test_content); CHECK(FileSystemManager::IsExecutable(*blob_path)); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/file_system/resolve_symlinks_map.test.cpp000066400000000000000000000223141516554100600330470ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/file_system/symlinks/resolve_symlinks_map.hpp" #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/file_system/symlinks/pragma_special.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "test/utils/shell_quoting.hpp" namespace { // see create_fs_test_git_bundle_symlinks.sh auto const kBundleSymPath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kTreeSymId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kFooId = std::string{"19102815663d23f8b75a47e7a01965dcdc96468c"}; auto const kBarId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kBazLinkId = std::string{"3f9538666251333f5fa519e01eb267d371ca9c78"}; auto const kBazSymId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"}; auto const kBazBarLinkId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; auto const kFooLinkId = std::string{"b24736f10d3c60015386047ebc98b4ab63056041"}; // see create_fs_test_git_bundle.sh auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo.bundle"}; auto const kBazId = std::string{"27b32561185c2825150893774953906c6daa6798"}; [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/file_system"; } [[nodiscard]] auto CreateTestRepo(bool is_bare = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] auto CreateTestRepoSymlinks(bool is_bare = false) -> std::optional { static std::atomic counter{}; auto repo_path = GetTestDir() / "test_repo_symlinks" / std::filesystem::path{std::to_string(counter++)}.filename(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundleSymPath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } // The checkout will make the content available, as well as the HEAD ref [[nodiscard]] auto CreateTestRepoSymlinksWithCheckout( bool is_bare = false) noexcept -> std::optional { auto repo_path = CreateTestRepoSymlinks(is_bare); REQUIRE(repo_path); auto cmd = fmt::format("git --git-dir={} --work-tree={} checkout master", QuoteForShell(is_bare ? repo_path->string() : (*repo_path / ".git").string()), QuoteForShell(repo_path->string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } } // namespace TEST_CASE("Resolve symlinks", "[resolve_symlinks_map]") { // non-bare repo with symlinks as source auto source_repo_path = CreateTestRepoSymlinksWithCheckout(); REQUIRE(source_repo_path); auto source_cas = GitCAS::Open(*source_repo_path); REQUIRE(source_cas); auto resolve_symlinks_map = CreateResolveSymlinksMap(); SECTION("Source repo is target repo") { constexpr auto kNumCases = 3; std::vector expected = { {kFooId, ObjectType::File, "baz/foo"}, {kBazBarLinkId, ObjectType::Symlink, "bar_l"}, {kBazId, ObjectType::Tree, "."}}; auto error = false; auto error_msg = std::string("NONE"); { TaskSystem ts; resolve_symlinks_map.ConsumeAfterKeysReady( &ts, {GitObjectToResolve(kTreeSymId, "foo_l", PragmaSpecial::ResolveCompletely, /*known_info=*/std::nullopt, source_cas, source_cas), GitObjectToResolve(kBazSymId, "bar_l", PragmaSpecial::ResolvePartially, /*known_info=*/std::nullopt, source_cas, source_cas), GitObjectToResolve(kBazSymId, ".", PragmaSpecial::Ignore, /*known_info=*/std::nullopt, source_cas, source_cas)}, [&expected, &source_cas](auto const& values) { for (std::size_t i = 0; i < kNumCases; ++i) { auto const& res = ResolvedGitObject{*values[i]}; CHECK(res.id == expected[i].id); CHECK(res.type == expected[i].type); CHECK(res.path == expected[i].path); // the object needs to be present in target repo CHECK( source_cas->ReadHeader(res.id, /*is_hex_id=*/true)); } }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK_FALSE(error); CHECK(error_msg == "NONE"); } SECTION("Bare target repo") { auto target_repo_path = CreateTestRepo(true); REQUIRE(target_repo_path); auto target_cas = GitCAS::Open(*target_repo_path); REQUIRE(target_cas); constexpr auto kNumCases = 3; std::vector expected = { {kFooId, ObjectType::File, "baz/foo"}, {kBazBarLinkId, ObjectType::Symlink, "bar_l"}, {kBazId, ObjectType::Tree, "."}}; auto error = false; auto error_msg = std::string("NONE"); { TaskSystem ts; resolve_symlinks_map.ConsumeAfterKeysReady( &ts, {GitObjectToResolve(kTreeSymId, "foo_l", PragmaSpecial::ResolveCompletely, /*known_info=*/std::nullopt, source_cas, target_cas), GitObjectToResolve(kBazSymId, "bar_l", PragmaSpecial::ResolvePartially, /*known_info=*/std::nullopt, source_cas, target_cas), GitObjectToResolve(kBazSymId, ".", PragmaSpecial::Ignore, /*known_info=*/std::nullopt, source_cas, target_cas)}, [&expected, &target_cas](auto const& values) { for (std::size_t i = 0; i < kNumCases; ++i) { auto const& res = ResolvedGitObject{*values[i]}; CHECK(res.id == expected[i].id); CHECK(res.type == expected[i].type); CHECK(res.path == expected[i].path); // the object needs to be present in target repo CHECK( target_cas->ReadHeader(res.id, /*is_hex_id=*/true)); } }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } CHECK_FALSE(error); CHECK(error_msg == "NONE"); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/000077500000000000000000000000001516554100600257715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/TARGETS000066400000000000000000000073531516554100600270350ustar00rootroot00000000000000{ "graph_traverser_tests": { "type": ["@", "rules", "CC", "library"] , "name": ["graph_traverser_tests"] , "hdrs": ["graph_traverser.test.hpp"] , "deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "cli"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/execution_api/common", "api_bundle"] , ["@", "src", "src/buildtool/execution_api/local", "config"] , ["@", "src", "src/buildtool/execution_api/local", "context"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/execution_api/remote", "context"] , ["@", "src", "src/buildtool/execution_engine/executor", "context"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "jsonfs"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/graph_traverser", "graph_traverser"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "test_hash_function_type"] ] , "stage": ["test", "buildtool", "graph_traverser"] } , "graph_traverser_local": { "type": ["@", "rules", "CC/test", "test"] , "name": ["graph_traverser_local"] , "srcs": ["graph_traverser_local.test.cpp"] , "data": ["test_data"] , "private-deps": [ "graph_traverser_tests" , ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "graph_traverser"] } , "graph_traverser_remote": { "type": ["utils/remote_execution", "CC test"] , "name": ["graph_traverser_remote"] , "srcs": ["graph_traverser_remote.test.cpp"] , "data": ["test_data"] , "private-deps": [ "graph_traverser_tests" , ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/storage", "backend_description"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] ] , "stage": ["test", "buildtool", "graph_traverser"] } , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [["TREE", null, "data"]] , "stage": ["test", "buildtool", "graph_traverser"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["graph_traverser"] , "deps": ["graph_traverser_local", "graph_traverser_remote"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/000077500000000000000000000000001516554100600267025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/copy_local_file/000077500000000000000000000000001516554100600320255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/copy_local_file/_entry_points000066400000000000000000000002231516554100600346410ustar00rootroot00000000000000{ "copied.hpp": { "type": "LOCAL", "data": { "path": "copy_me.hpp", "repository": "" } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/copy_local_file/copy_me.hpp000066400000000000000000000011551516554100600341730ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. graph_description000066400000000000000000000000701516554100600353720ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/copy_local_file{ "blobs": [], "trees": {}, "actions": {} } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world/000077500000000000000000000000001516554100600324025ustar00rootroot00000000000000_entry_points000066400000000000000000000002401516554100600351360ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world{ "greeting_output": { "type": "ACTION", "data": { "id": "make_output", "path": "greeting_output" } } } _entry_points_ctimes000066400000000000000000000002161516554100600365050ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world{ "ctimes": { "type": "ACTION", "data": { "id": "list_ctimes", "path": "ctimes" } } } _entry_points_stripped000066400000000000000000000002611516554100600370530ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world{ "stripped_greeting_output": { "type": "ACTION", "data": { "id": "strip_time", "path": "stripped_greeting_output" } } } graph_description000066400000000000000000000041661516554100600357610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world{ "actions": { "make_exe": { "output": [ "hello_world" ], "command": [ "c++", "hello_world.cpp", "-o", "hello_world" ], "input": { "hello_world.cpp": { "type": "LOCAL", "data": { "path": "hello_world.cpp", "repository": "" } } } }, "make_output": { "output": [ "greeting_output" ], "command": [ "/bin/sh", "-c", "set -e\n./hello_world > greeting_output" ], "input": { "hello_world": { "type": "ACTION", "data": { "id": "make_exe", "path": "hello_world" } } } }, "strip_time": { "output": [ "stripped_greeting_output" ], "command": [ "/bin/sh", "-c", "set -e\nhead -n1 greeting_output > stripped_greeting_output" ], "input": { "greeting_output": { "type": "ACTION", "data": { "id": "make_output", "path": "greeting_output" } } } }, "list_ctimes": { "output": [ "ctimes" ], "command": [ "/bin/sh", "-c", "set -e\nls --full-time --time ctime > ctimes" ], "input": { "stripped_greeting_output": { "type": "ACTION", "data": { "id": "strip_time", "path": "stripped_greeting_output" } } } } }, "blobs": [], "trees": {} } hello_world.cpp000066400000000000000000000013661516554100600353470ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/flaky_hello_world// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include int main() { std::cout << "Hello, World!" << std::endl << "It's now " << __TIME__ << std::endl; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message/000077500000000000000000000000001516554100600337525ustar00rootroot00000000000000_entry_points000066400000000000000000000002401516554100600365060ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message{ "greeting_output": { "type": "ACTION", "data": { "id": "make_output", "path": "greeting_output" } } } _entry_points_get_executable000077500000000000000000000002241516554100600415530ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message{ "executable": { "type": "ACTION", "data": { "id": "make_exe", "path": "hello_world" } } } _entry_points_upload_source000066400000000000000000000002371516554100600414400ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message{ "local_src_file.cpp": { "type": "LOCAL", "data": { "path": "hello_world.cpp", "repository": "" } } } graph_description000066400000000000000000000020561516554100600373250ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message{ "actions": { "make_exe": { "output": [ "hello_world" ], "command": [ "c++", "hello_world.cpp", "-o", "hello_world" ], "input": { "hello_world.cpp": { "type": "LOCAL", "data": { "path": "hello_world.cpp", "repository": "" } } } }, "make_output": { "output": [ "greeting_output" ], "command": [ "/bin/sh", "-c", "set -e\n./hello_world > greeting_output\nls -alR\n" ], "input": { "hello_world": { "type": "ACTION", "data": { "id": "make_exe", "path": "hello_world" } } } } }, "blobs": [], "trees": {} } hello_world.cpp000066400000000000000000000013001516554100600367030ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_copy_message// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include int main() { std::cout << "Hello, World!" << std::endl; } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_known_source/000077500000000000000000000000001516554100600340105ustar00rootroot00000000000000_entry_points000066400000000000000000000002401516554100600365440ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_known_source{ "greeting_output": { "type": "ACTION", "data": { "id": "make_output", "path": "greeting_output" } } } graph_description000066400000000000000000000021521516554100600373600ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_known_source{ "actions": { "make_exe": { "output": [ "hello_world" ], "command": [ "c++", "hello_world.cpp", "-o", "hello_world" ], "input": { "hello_world.cpp": { "type": "KNOWN", "data": { "id": "435ae8056332c434c01b217bed8b658d62df2568", "size": 704, "file_type": "f" } } } }, "make_output": { "output": [ "greeting_output" ], "command": [ "/bin/sh", "-c", "set -e\n./hello_world > greeting_output\nls -alR\n" ], "input": { "hello_world": { "type": "ACTION", "data": { "id": "make_exe", "path": "hello_world" } } } } }, "blobs": [], "trees": {} } graph_description_compatible000066400000000000000000000022021516554100600415530ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/hello_world_known_source{ "actions": { "make_exe": { "output": [ "hello_world" ], "command": [ "c++", "hello_world.cpp", "-o", "hello_world" ], "input": { "hello_world.cpp": { "type": "KNOWN", "data": { "id": "ec25802f5e7bbb752c7416fdc90ee62d1be342de63bdda031eec4b9e92d775e7", "size": 704, "file_type": "f" } } } }, "make_output": { "output": [ "greeting_output" ], "command": [ "/bin/sh", "-c", "set -e\n./hello_world > greeting_output\nls -alR\n" ], "input": { "hello_world": { "type": "ACTION", "data": { "id": "make_exe", "path": "hello_world" } } } } }, "blobs": [], "trees": {} } sequence_printer_build_library_only/000077500000000000000000000000001516554100600361425ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data_entry_points000066400000000000000000000002251516554100600407600ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only{ "sequences.a": { "type": "ACTION", "data": { "id": "sequences/link", "path": "lib.a" } } } _entry_points_full_build000066400000000000000000000002311516554100600431560ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only{ "sequence_printer.out": { "type": "ACTION", "data": { "id": "main/link", "path": "a.out" } } } graph_description000066400000000000000000000114611516554100600415740ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only{ "blobs": [], "trees": {}, "actions": { "sequences/random_dna_sequence/compile": { "input": { "sequence.hpp": { "type": "LOCAL", "data": { "path": "sequences/sequence.hpp", "repository": "" } }, "random_dna_sequence.hpp": { "type": "LOCAL", "data": { "path": "sequences/random_dna_sequence.hpp", "repository": "" } }, "random_dna_sequence.cpp": { "type": "LOCAL", "data": { "path": "sequences/random_dna_sequence.cpp", "repository": "" } } }, "output": [ "obj.o" ], "command": [ "c++", "-o", "obj.o", "-c", "random_dna_sequence.cpp" ] }, "sequences/fibonacci/compile": { "input": { "sequence.hpp": { "type": "LOCAL", "data": { "path": "sequences/sequence.hpp", "repository": "" } }, "fibonacci.hpp": { "type": "LOCAL", "data": { "path": "sequences/fibonacci.hpp", "repository": "" } }, "fibonacci.cpp": { "type": "LOCAL", "data": { "path": "sequences/fibonacci.cpp", "repository": "" } } }, "output": [ "obj.o" ], "command": [ "c++", "-o", "obj.o", "-c", "fibonacci.cpp" ] }, "sequences/link": { "input": { "obj1.o": { "type": "ACTION", "data": { "id": "sequences/random_dna_sequence/compile", "path": "obj.o" } }, "obj2.o": { "type": "ACTION", "data": { "id": "sequences/fibonacci/compile", "path": "obj.o" } } }, "output": [ "lib.a" ], "command": [ "ar", "cqs", "lib.a", "obj1.o", "obj2.o" ] }, "main/compile": { "input": { "fibonacci.hpp": { "type": "LOCAL", "data": { "path": "sequences/fibonacci.hpp", "repository": "" } }, "random_dna_sequence.hpp": { "type": "LOCAL", "data": { "path": "sequences/random_dna_sequence.hpp", "repository": "" } }, "sequence.hpp": { "type": "LOCAL", "data": { "path": "sequences/sequence.hpp", "repository": "" } }, "printer.hpp": { "type": "LOCAL", "data": { "path": "printer/printer.hpp", "repository": "" } }, "main.cpp": { "type": "LOCAL", "data": { "path": "main.cpp", "repository": "" } } }, "output": [ "obj.o" ], "command": [ "c++", "-o", "obj.o", "-c", "main.cpp" ] }, "main/link": { "input": { "lib1.a": { "type": "ACTION", "data": { "id": "sequences/link", "path": "lib.a" } }, "obj1.o": { "type": "ACTION", "data": { "id": "main/compile", "path": "obj.o" } } }, "output": [ "a.out" ], "command": [ "c++", "-o", "a.out", "obj1.o", "lib1.a" ] } } } main.cpp000066400000000000000000000024421516554100600375740ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "fibonacci.hpp" #include "printer.hpp" #include "random_dna_sequence.hpp" int main() { Printer printer; Fibonacci fib; std::cout << "PRINT 10 following terms of Fibonacci sequence starting with 0 1" << std::endl; printer.print(fib, 10U); std::cout << std::endl; Fibonacci fib2_5{2, 5}; std::cout << "PRINT 8 following terms of Fibonacci sequence starting with 2 5" << std::endl; printer.print(fib2_5, 8U); std::cout << std::endl; RandomDNASequence piece_of_something; std::cout << "PRINT a random dna sequence of length 3" << std::endl; printer.print(piece_of_something, 30U); } printer/000077500000000000000000000000001516554100600376255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_onlyprinter.hpp000066400000000000000000000020501516554100600420160ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/printer// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include class Printer { public: template void print(SequenceT& seq, unsigned int number_of_terms) { if (number_of_terms == 0) { std::cout << std::endl; return; } for (unsigned int i = 0; i < number_of_terms - 1; ++i) { std::cout << seq.next() << seq.separator(); } std::cout << seq.next() << std::endl; } }; sequences/000077500000000000000000000000001516554100600401355ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_onlyfibonacci.cpp000066400000000000000000000017161516554100600425630ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/sequences// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "fibonacci.hpp" Fibonacci::Fibonacci() : second_prev_{0}, prev_{1} {} Fibonacci::Fibonacci(int zero_th, int first) : second_prev_{zero_th}, prev_{first} {} int Fibonacci::next() { int next = second_prev_ + prev_; second_prev_ = prev_; prev_ = next; return next; } std::string Fibonacci::separator() { return ", "; } fibonacci.hpp000066400000000000000000000016501516554100600425650ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/sequences// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include "sequence.hpp" class Fibonacci : public Sequence { public: Fibonacci(); Fibonacci(int zeroth, int first); ~Fibonacci() override = default; int next() override; std::string separator() override; private: int second_prev_; int prev_; }; random_dna_sequence.cpp000066400000000000000000000022431516554100600446340ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/sequences// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "random_dna_sequence.hpp" RandomDNASequence::RandomDNASequence() : eng_(static_cast( std::chrono::system_clock::now().time_since_epoch().count())), dist_(0, 3) {} RandomDNASequence::RandomDNASequence(unsigned int seed) : eng_(seed), dist_(0, 3) {} char RandomDNASequence::next() { int option = dist_(eng_); if (option == 0) return 'A'; if (option == 1) return 'C'; if (option == 2) return 'G'; return 'T'; } std::string RandomDNASequence::separator() { return ""; } random_dna_sequence.hpp000066400000000000000000000020361516554100600446410ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/sequences// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "sequence.hpp" class RandomDNASequence : public Sequence { public: RandomDNASequence(); explicit RandomDNASequence(unsigned int seed); ~RandomDNASequence() override = default; char next() override; std::string separator() override; private: std::default_random_engine eng_; std::uniform_int_distribution<> dist_; }; sequence.hpp000066400000000000000000000015051516554100600424570ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/sequence_printer_build_library_only/sequences// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include template class Sequence { public: typedef T value_type; virtual T next() = 0; virtual std::string separator() = 0; virtual ~Sequence() = default; }; just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_env_variables/000077500000000000000000000000001516554100600323765ustar00rootroot00000000000000_entry_points000066400000000000000000000002111516554100600351300ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_env_variables{ "out": { "type": "ACTION", "data": { "id": "write_to_file", "path": "out" } } }graph_description000066400000000000000000000006461516554100600357540ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_env_variables{ "blobs": [], "trees": {}, "actions": { "write_to_file": { "output": ["out"], "command": [ "/bin/sh", "-c", "set -e\necho -n ${MYCONTENT} > out" ], "env": { "MYCONTENT": "content from environment variable", "UNUSED_VAR": "nothing important" } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_nested_trees/000077500000000000000000000000001516554100600322425ustar00rootroot00000000000000_entry_points000066400000000000000000000002231516554100600347770ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_nested_trees{ "statement": { "type": "ACTION", "data": { "id": "write_test", "path": "statement" } } } graph_description000066400000000000000000000025061516554100600356150ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_nested_trees{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": { "tree-0": { "subject": { "type": "KNOWN", "data": { "id": "a2a3f4f1e30c488bfbd52aabfbcfcc1f5822158d", "size": 4, "file_type": "f" } } }, "tree-1": { "nested": { "type": "TREE", "data": { "id": "tree-0" } } } }, "actions": { "write_test": { "input": { "main": { "type": "TREE", "data": { "id": "tree-1" } }, "thing": { "type": "KNOWN", "data": { "id": "d4d7eecc25bcbd902b0b97a97e8e2e478c97454b", "size": 35, "file_type": "f" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat main/nested/subject) is a $(cat thing)\" > statement" ] } } } graph_description_compatible000066400000000000000000000025661516554100600400220ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_nested_trees{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": { "tree-0": { "subject": { "type": "KNOWN", "data": { "id": "1eb79602411ef02cf6fe117897015fff89f80face4eccd50425c45149b148408", "size": 4, "file_type": "f" } } }, "tree-1": { "nested": { "type": "TREE", "data": { "id": "tree-0" } } } }, "actions": { "write_test": { "input": { "main": { "type": "TREE", "data": { "id": "tree-1" } }, "thing": { "type": "KNOWN", "data": { "id": "0bbd636ca17d06a5b630fe3f19844192f57fb70ae9b24686cc8a2fa7e48c9b77", "size": 35, "file_type": "f" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat main/nested/subject) is a $(cat thing)\" > statement" ] } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_trees/000077500000000000000000000000001516554100600307005ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_trees/_entry_points000066400000000000000000000002221516554100600335130ustar00rootroot00000000000000{ "statement": { "type": "ACTION", "data": { "id": "read_test", "path": "statement" } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_trees/graph_description000066400000000000000000000034101516554100600343250ustar00rootroot00000000000000{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": { "tree-0": { "subject": { "type": "KNOWN", "data": { "id": "a2a3f4f1e30c488bfbd52aabfbcfcc1f5822158d", "size": 4, "file_type": "f" } }, "thing": { "type": "KNOWN", "data": { "id": "d4d7eecc25bcbd902b0b97a97e8e2e478c97454b", "size": 35, "file_type": "f" } } }, "tree-1": { "test_data": { "type": "ACTION", "data": { "id": "write_test", "path": "statement" } } } }, "actions": { "write_test": { "input": { ".": { "type": "TREE", "data": { "id": "tree-0" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat subject) is a $(cat thing)\" > statement" ] }, "read_test": { "input": { "data": { "type": "TREE", "data": { "id": "tree-1" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "cat data/test_data > statement" ] } } } graph_description_compatible000066400000000000000000000034701516554100600364530ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_trees{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": { "tree-0": { "subject": { "type": "KNOWN", "data": { "id": "1eb79602411ef02cf6fe117897015fff89f80face4eccd50425c45149b148408", "size": 4, "file_type": "f" } }, "thing": { "type": "KNOWN", "data": { "id": "0bbd636ca17d06a5b630fe3f19844192f57fb70ae9b24686cc8a2fa7e48c9b77", "size": 35, "file_type": "f" } } }, "tree-1": { "test_data": { "type": "ACTION", "data": { "id": "write_test", "path": "statement" } } } }, "actions": { "write_test": { "input": { ".": { "type": "TREE", "data": { "id": "tree-0" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat subject) is a $(cat thing)\" > statement" ] }, "read_test": { "input": { "data": { "type": "TREE", "data": { "id": "tree-1" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "cat data/test_data > statement" ] } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_uploaded_blobs/000077500000000000000000000000001516554100600325345ustar00rootroot00000000000000_entry_points000066400000000000000000000002231516554100600352710ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_uploaded_blobs{ "statement": { "type": "ACTION", "data": { "id": "write_test", "path": "statement" } } } graph_description000066400000000000000000000017301516554100600361050ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_uploaded_blobs{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": {}, "actions": { "write_test": { "input": { "subject": { "type": "KNOWN", "data": { "id": "a2a3f4f1e30c488bfbd52aabfbcfcc1f5822158d", "size": 4, "file_type": "f" } }, "thing": { "type": "KNOWN", "data": { "id": "d4d7eecc25bcbd902b0b97a97e8e2e478c97454b", "size": 35, "file_type": "f" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat subject) is a $(cat thing)\" > statement" ] } } } graph_description_compatible000066400000000000000000000020101516554100600402740ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/data/use_uploaded_blobs{ "blobs": ["test to check if blobs are uploaded", "this"], "trees": {}, "actions": { "write_test": { "input": { "subject": { "type": "KNOWN", "data": { "id": "1eb79602411ef02cf6fe117897015fff89f80face4eccd50425c45149b148408", "size": 4, "file_type": "f" } }, "thing": { "type": "KNOWN", "data": { "id": "0bbd636ca17d06a5b630fe3f19844192f57fb70ae9b24686cc8a2fa7e48c9b77", "size": 35, "file_type": "f" } } }, "output": [ "statement" ], "command": [ "/bin/sh", "-c", "set -e\necho -n \"$(cat subject) is a $(cat thing)\" > statement" ] } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/graph_traverser.test.hpp000066400000000000000000000733201516554100600326630ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_BUILDTOOL_GRAPH_GRAVERSER_GRAPH_TRAVERSER_TEST_HPP #define INCLUDED_SRC_TEST_BUILDTOOL_GRAPH_GRAVERSER_GRAPH_TRAVERSER_TEST_HPP #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/execution_engine/executor/context.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/jsonfs.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" // NOLINTNEXTLINE(google-build-namespaces) namespace { class TestProject { public: struct CommandLineArguments { GraphTraverser::CommandLineArguments gtargs; nlohmann::json artifacts; std::filesystem::path graph_description; explicit CommandLineArguments( GraphTraverser::CommandLineArguments gtargs) : gtargs{std::move(std::move(gtargs))} {} }; // NOLINTNEXTLINE(modernize-pass-by-value) explicit TestProject(std::string const& example_name) : example_name_{example_name}, root_dir_{kWorkspacePrefix / example_name_} { SetupConfig(); } explicit TestProject(std::string&& example_name) : example_name_{std::move(example_name)}, root_dir_{kWorkspacePrefix / example_name_} { SetupConfig(); } /// \brief Get command line arguments parsing entry points file in /// data//, where /// takes "_entry_points" as default value auto CmdLineArgs(std::string const& entry_points_filename = kDefaultEntryPointsFileName) -> CommandLineArguments { auto const entry_points_file = root_dir_ / entry_points_filename; if (not FileSystemManager::IsFile(entry_points_file)) { Logger::Log( LogLevel::Error, "file with entry points for graph_traverser tests can not be " "found in path {}", entry_points_file.string()); std::exit(EXIT_FAILURE); } auto const entry_points_json = Json::ReadFile(entry_points_file); if (not entry_points_json.has_value()) { Logger::Log(LogLevel::Error, "can not read {} for graph_traverser tests", entry_points_file.string()); std::exit(EXIT_FAILURE); } return GenerateFromEntryPoints(*entry_points_json); } auto GetRepoConfig() -> RepositoryConfig* { return &repo_config_; } private: static inline std::filesystem::path const kOutputDirPrefix = FileSystemManager::GetCurrentDirectory() / "./tmp-"; static inline std::filesystem::path const kWorkspacePrefix = FileSystemManager::GetCurrentDirectory() / "test/buildtool/graph_traverser/data/"; static inline std::string const kDefaultEntryPointsFileName = "_entry_points"; std::string example_name_; std::filesystem::path root_dir_; RepositoryConfig repo_config_{}; void SetupConfig() { auto info = RepositoryConfig::RepositoryInfo{FileRoot{root_dir_}}; repo_config_.SetInfo("", std::move(info)); } auto GenerateFromEntryPoints(nlohmann::json const& entry_points) -> CommandLineArguments { static int id{}; GraphTraverser::CommandLineArguments gtargs{0, {}, {}, {}}; CommandLineArguments clargs{gtargs}; clargs.artifacts = entry_points; auto const comp_graph = root_dir_ / "graph_description_compatible"; if (not ProtocolTraits::IsNative( TestHashType::ReadFromEnvironment()) and FileSystemManager::Exists(comp_graph)) { clargs.graph_description = comp_graph; } else { clargs.graph_description = root_dir_ / "graph_description"; } clargs.gtargs.jobs = std::max(1U, std::thread::hardware_concurrency()); clargs.gtargs.stage = StageArguments{ kOutputDirPrefix / (example_name_ + std::to_string(id++))}; return clargs; } }; [[nodiscard]] inline auto CreateLocalExecConfig() noexcept -> LocalExecutionConfig { std::vector launcher{"env"}; auto* env_path = std::getenv("PATH"); if (env_path != nullptr) { launcher.emplace_back(std::string{"PATH="} + std::string{env_path}); } else { launcher.emplace_back("PATH=/bin:/usr/bin"); } LocalExecutionConfig::Builder builder; if (auto config = builder.SetLauncher(std::move(launcher)).Build()) { return *std::move(config); } Logger::Log(LogLevel::Error, "Failure setting the local launcher."); std::exit(EXIT_FAILURE); } } // namespace [[maybe_unused]] static void TestHelloWorldCopyMessage( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("hello_world_copy_message"); auto const local_exec_config = CreateLocalExecConfig(); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const contents = FileSystemManager::ReadFile(result->output_paths[0]); CHECK(contents.has_value()); CHECK(contents == "Hello, World!\n"); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 2); CHECK(stats.ActionsCachedCounter() == 0); } SECTION("Executable is retrieved as executable") { auto const clargs_exec = p.CmdLineArgs("_entry_points_get_executable"); GraphTraverser const gt_get_exec{clargs_exec.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const exec_result = gt_get_exec.BuildAndStage( clargs_exec.graph_description, clargs_exec.artifacts); REQUIRE(exec_result); REQUIRE(exec_result->output_paths.size() == 1); auto const exec_path = exec_result->output_paths[0]; CHECK(FileSystemManager::IsFile(exec_path)); CHECK(FileSystemManager::IsExecutable(exec_path)); CHECK(FileSystemManager::Type(exec_path) == ObjectType::Executable); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 3); // One more action queued CHECK(stats.ActionsCachedCounter() == 1); // But that action was cached } } } [[maybe_unused]] static void TestCopyLocalFile( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("copy_local_file"); auto const local_exec_config = CreateLocalExecConfig(); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 0); CHECK(stats.ActionsCachedCounter() == 0); } } [[maybe_unused]] static void TestSequencePrinterBuildLibraryOnly( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("sequence_printer_build_library_only"); auto const local_exec_config = CreateLocalExecConfig(); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const clargs_full_build = p.CmdLineArgs("_entry_points_full_build"); GraphTraverser const gt_full_build{clargs_full_build.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const full_build_result = gt_full_build.BuildAndStage( clargs_full_build.graph_description, clargs_full_build.artifacts); REQUIRE(full_build_result); REQUIRE(full_build_result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(full_build_result->output_paths[0])); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 8); CHECK(stats.ActionsCachedCounter() == 3); } else { CHECK(stats.ActionsCachedCounter() > 0); } } [[maybe_unused]] static void TestHelloWorldWithKnownSource( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject full_hello_world("hello_world_copy_message"); auto const local_exec_config = CreateLocalExecConfig(); auto const clargs_update_cpp = full_hello_world.CmdLineArgs("_entry_points_upload_source"); Statistics stats{}; Progress progress{}; // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const full_apis = ApiBundle::Create( &local_context, &remote_context, full_hello_world.GetRepoConfig()); ExecutionContext const full_context{ .repo_config = full_hello_world.GetRepoConfig(), .apis = &full_apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const gt_upload{clargs_update_cpp.gtargs, &full_context, [](auto /*done*/, auto /*cv*/) {}}; auto const cpp_result = gt_upload.BuildAndStage( clargs_update_cpp.graph_description, clargs_update_cpp.artifacts); REQUIRE(cpp_result); REQUIRE(cpp_result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(cpp_result->output_paths[0])); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 0); CHECK(stats.ActionsCachedCounter() == 0); } TestProject hello_world_known_cpp("hello_world_known_source"); auto const clargs = hello_world_known_cpp.CmdLineArgs(); auto const apis_known = ApiBundle::Create( &local_context, &remote_context, hello_world_known_cpp.GetRepoConfig()); ExecutionContext const context_known{ .repo_config = hello_world_known_cpp.GetRepoConfig(), .apis = &apis_known, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser const gt{ clargs.gtargs, &context_known, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 2); CHECK(stats.ActionsCachedCounter() == 0); } else { CHECK(stats.ActionsQueuedCounter() >= 2); } } static void TestBlobsUploadedAndUsed( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("use_uploaded_blobs"); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const contents = FileSystemManager::ReadFile(result->output_paths[0]); CHECK(contents.has_value()); CHECK(contents == "this is a test to check if blobs are uploaded"); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 1); CHECK(stats.ActionsCachedCounter() == 0); } else { CHECK(stats.ActionsQueuedCounter() >= 1); } } static void TestEnvironmentVariablesSetAndUsed( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("use_env_variables"); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const contents = FileSystemManager::ReadFile(result->output_paths[0]); CHECK(contents.has_value()); CHECK(contents == "content from environment variable"); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 1); CHECK(stats.ActionsCachedCounter() == 0); } else { CHECK(stats.ActionsQueuedCounter() >= 1); } } static void TestTreesUsed( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("use_trees"); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const contents = FileSystemManager::ReadFile(result->output_paths[0]); CHECK(contents.has_value()); CHECK(contents == "this is a test to check if blobs are uploaded"); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 2); CHECK(stats.ActionsCachedCounter() == 0); } else { CHECK(stats.ActionsQueuedCounter() >= 2); } } static void TestNestedTreesUsed( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool is_hermetic = true) { TestProject p("use_nested_trees"); auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; GraphTraverser gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); CHECK(FileSystemManager::IsFile(result->output_paths[0])); auto const contents = FileSystemManager::ReadFile(result->output_paths[0]); CHECK(contents.has_value()); CHECK(contents == "this is a test to check if blobs are uploaded"); if (is_hermetic) { CHECK(stats.ActionsQueuedCounter() == 1); CHECK(stats.ActionsCachedCounter() == 0); } else { CHECK(stats.ActionsQueuedCounter() >= 1); } } static void TestFlakyHelloWorldDetected( StorageConfig const& storage_config, Storage const& storage, gsl::not_null const& auth, gsl::not_null const& remote_config, bool /*is_hermetic*/ = true) { TestProject p("flaky_hello_world"); Statistics stats{}; Progress progress{}; auto const local_exec_config = CreateLocalExecConfig(); // pack the local context instances to be passed to ApiBundle LocalContext const local_context{.exec_config = &local_exec_config, .storage_config = &storage_config, .storage = &storage}; RetryConfig retry_config{}; // default retry config // pack the remote context instances to be passed to ApiBundle RemoteContext const remote_context{.auth = auth, .retry_config = &retry_config, .exec_config = remote_config}; auto const apis = ApiBundle::Create(&local_context, &remote_context, p.GetRepoConfig()); ExecutionContext const exec_context{.repo_config = p.GetRepoConfig(), .apis = &apis, .remote_context = &remote_context, .statistics = &stats, .progress = &progress, .profile = std::nullopt}; { auto clargs = p.CmdLineArgs("_entry_points_ctimes"); GraphTraverser const gt{ clargs.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; auto const result = gt.BuildAndStage(clargs.graph_description, clargs.artifacts); REQUIRE(result); REQUIRE(result->output_paths.size() == 1); } using namespace std::chrono_literals; std::this_thread::sleep_for(1s); // make_exe[flaky]->make_output[miss] auto clargs_output = p.CmdLineArgs(); clargs_output.gtargs.rebuild = RebuildArguments{}; GraphTraverser const gt_output{ clargs_output.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; REQUIRE(gt_output.BuildAndStage(clargs_output.graph_description, clargs_output.artifacts)); CHECK(stats.ActionsFlakyCounter() == 1); CHECK(stats.RebuiltActionComparedCounter() == 1); CHECK(stats.RebuiltActionMissingCounter() == 1); stats.Reset(); // make_exe[flaky]->make_output[miss]->strip_time [miss] auto clargs_stripped = p.CmdLineArgs("_entry_points_stripped"); clargs_stripped.gtargs.rebuild = RebuildArguments{}; GraphTraverser const gt_stripped{clargs_stripped.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; REQUIRE(gt_stripped.BuildAndStage(clargs_stripped.graph_description, clargs_stripped.artifacts)); CHECK(stats.ActionsFlakyCounter() == 1); CHECK(stats.RebuiltActionComparedCounter() == 1); CHECK(stats.RebuiltActionMissingCounter() == 2); stats.Reset(); // make_exe[flaky]->make_output[miss]->strip_time[miss]->list_ctimes [flaky] auto clargs_ctimes = p.CmdLineArgs("_entry_points_ctimes"); clargs_ctimes.gtargs.rebuild = RebuildArguments{}; GraphTraverser const gt_ctimes{ clargs_ctimes.gtargs, &exec_context, [](auto /*done*/, auto /*cv*/) {}}; REQUIRE(gt_ctimes.BuildAndStage(clargs_ctimes.graph_description, clargs_ctimes.artifacts)); CHECK(stats.ActionsFlakyCounter() == 2); CHECK(stats.RebuiltActionComparedCounter() == 2); CHECK(stats.RebuiltActionMissingCounter() == 2); } #endif // INCLUDED_SRC_TEST_BUILDTOOL_GRAPH_GRAVERSER_GRAPH_TRAVERSER_TEST_HPP just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/graph_traverser_local.test.cpp000066400000000000000000000112651516554100600340300ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/buildtool/graph_traverser/graph_traverser.test.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" TEST_CASE("Local: Output created when entry point is local artifact", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestCopyLocalFile(storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Output created and contents are correct", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestHelloWorldCopyMessage( storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Actions are not re-run", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestSequencePrinterBuildLibraryOnly( storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: KNOWN artifact", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestHelloWorldWithKnownSource( storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Blobs uploaded and correctly used", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestBlobsUploadedAndUsed( storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Environment variables are set and used", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestEnvironmentVariablesSetAndUsed( storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Trees correctly used", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestTreesUsed(storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Nested trees correctly used", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestNestedTreesUsed(storage_config.Get(), storage, &auth, &remote_config); } TEST_CASE("Local: Detect flaky actions", "[graph_traverser]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); Auth auth{}; /*no TLS needed*/ RemoteExecutionConfig remote_config{}; /*no remote*/ TestFlakyHelloWorldDetected( storage_config.Get(), storage, &auth, &remote_config); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/graph_traverser/graph_traverser_remote.test.cpp000066400000000000000000000203421516554100600342250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "test/buildtool/graph_traverser/graph_traverser.test.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" [[nodiscard]] static auto CreateStorageConfig( RemoteExecutionConfig const& remote_config) -> StorageConfig { auto cache_dir = FileSystemManager::GetCurrentDirectory() / "cache"; if (not FileSystemManager::RemoveDirectory(cache_dir) or not FileSystemManager::CreateDirectoryExclusive(cache_dir)) { Logger::Log(LogLevel::Error, "failed to create a test-local cache dir {}", cache_dir.string()); std::exit(EXIT_FAILURE); } auto backend_description = BackendDescription::Describe(remote_config.remote_address, remote_config.platform_properties, remote_config.dispatch); if (not backend_description) { Logger::Log(LogLevel::Error, std::move(backend_description).error()); std::exit(EXIT_FAILURE); } StorageConfig::Builder builder; auto config = builder.SetBuildRoot(cache_dir) .SetHashType(TestHashType::ReadFromEnvironment()) .SetBackendDescription(*std::move(backend_description)) .Build(); if (not config) { Logger::Log(LogLevel::Error, config.error()); std::exit(EXIT_FAILURE); } return *std::move(config); } TEST_CASE("Remote: Output created and contents are correct", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestHelloWorldCopyMessage(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Output created when entry point is local artifact", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestCopyLocalFile(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Actions are not re-run", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestSequencePrinterBuildLibraryOnly(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: KNOWN artifact", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestHelloWorldWithKnownSource(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Blobs uploaded and correctly used", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestBlobsUploadedAndUsed(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Environment variables are set and used", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestEnvironmentVariablesSetAndUsed(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Trees correctly used", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestTreesUsed(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Nested trees correctly used", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestNestedTreesUsed(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } TEST_CASE("Remote: Detect flaky actions", "[graph_traverser]") { auto remote_config = TestRemoteConfig::ReadFromEnvironment(); REQUIRE(remote_config); StorageConfig const storage_config = CreateStorageConfig(*remote_config); auto const storage = Storage::Create(&storage_config); auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); TestFlakyHelloWorldDetected(storage_config, storage, &*auth_config, &*remote_config, false /* not hermetic */); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/logging/000077500000000000000000000000001516554100600242215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/logging/TARGETS000066400000000000000000000017221516554100600252570ustar00rootroot00000000000000{ "logger": { "type": ["@", "rules", "CC/test", "test"] , "name": ["logger"] , "srcs": ["logger.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "logging"] } , "log_sink_file": { "type": ["@", "rules", "CC/test", "test"] , "name": ["log_sink_file"] , "srcs": ["log_sink_file.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "logging"] , "private-ldflags": ["-pthread"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["logging"] , "deps": ["log_sink_file", "logger"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/logging/log_sink_file.test.cpp000066400000000000000000000075231516554100600305160ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/logging/log_sink_file.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" [[nodiscard]] static auto NumberOfLines(std::filesystem::path const& file_path) -> int { std::ifstream file(file_path); std::string line{}; int number_of_lines{}; while (std::getline(file, line)) { ++number_of_lines; } return number_of_lines; } [[nodiscard]] static auto GetLines(std::filesystem::path const& file_path) -> std::vector { std::ifstream file(file_path); std::string line{}; std::vector lines{}; while (std::getline(file, line)) { lines.push_back(line); } return lines; } TEST_CASE("LogSinkFile", "[logging]") { LogConfig::SetSinks({LogSinkCmdLine::CreateFactory(false /*no color*/)}); // cleanup std::string filename{"test/test.log"}; CHECK(FileSystemManager::RemoveFile(filename)); REQUIRE(not FileSystemManager::IsFile(filename)); // create test log file REQUIRE(FileSystemManager::WriteFile("somecontent\n", filename)); REQUIRE(FileSystemManager::IsFile(filename)); CHECK(NumberOfLines(filename) == 1); SECTION("Overwrite mode") { LogSinkFile sink{filename, LogSinkFile::Mode::Overwrite}; sink.Emit(nullptr, LogLevel::Info, "first"); sink.Emit(nullptr, LogLevel::Info, "second"); sink.Emit(nullptr, LogLevel::Info, "third"); // read file and check line numbers CHECK(NumberOfLines(filename) == 3); } SECTION("Append mode") { LogSinkFile sink{filename, LogSinkFile::Mode::Append}; sink.Emit(nullptr, LogLevel::Info, "first"); sink.Emit(nullptr, LogLevel::Info, "second"); sink.Emit(nullptr, LogLevel::Info, "third"); // read file and check line numbers CHECK(NumberOfLines(filename) == 4); } SECTION("Thread-safety") { int const num_threads = 20; LogSinkFile sink{filename, LogSinkFile::Mode::Append}; // start threads, each emitting a log message std::vector threads{}; for (int id{}; id < num_threads; ++id) { threads.emplace_back( [&](int tid) { sink.Emit(nullptr, LogLevel::Info, "this is thread " + std::to_string(tid)); }, id); } // wait for threads to finish for (auto& thread : threads) { thread.join(); } // read file and check line numbers auto lines = GetLines(filename); CHECK(lines.size() == num_threads + 1); // check for corrupted content for (auto const& line : lines) { CHECK_THAT( line, Catch::Matchers::ContainsSubstring("somecontent") or Catch::Matchers::ContainsSubstring("this is thread")); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/logging/logger.test.cpp000066400000000000000000000360461516554100600271730ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/logging/logger.hpp" #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink.hpp" // Stores prints from test sink instances class TestPrints { struct PrintData { std::atomic counter; std::unordered_map> prints; }; public: static void Print(int sink_id, std::string const& print) noexcept { Data().prints[sink_id].push_back(print); } [[nodiscard]] static auto Read(int sink_id) noexcept -> std::vector { return Data().prints[sink_id]; } static void Clear() noexcept { Data().prints.clear(); Data().counter = 0; } static auto GetId() noexcept -> int { return Data().counter++; } private: [[nodiscard]] static auto Data() noexcept -> PrintData& { static PrintData instance{}; return instance; } }; // Test sink, prints to TestPrints depending on its own instance id. class LogSinkTest : public ILogSink { public: static auto CreateFactory() -> LogSinkFactory { return [] { return std::make_shared(); }; } LogSinkTest() noexcept : id_{TestPrints::GetId()} {} void Emit(Logger const* logger, LogLevel level, std::string const& msg) const noexcept final { auto prefix = LogLevelToString(level); if (logger != nullptr) { prefix += " (" + logger->Name() + ")"; } TestPrints::Print(id_, prefix + ": " + msg); } private: int id_{}; }; class OneGlobalSinkFixture { public: OneGlobalSinkFixture() { TestPrints::Clear(); LogConfig::SetLogLimit(LogLevel::Info); LogConfig::SetSinks({LogSinkTest::CreateFactory()}); } }; class TwoGlobalSinksFixture : public OneGlobalSinkFixture { public: TwoGlobalSinksFixture() { LogConfig::AddSink(LogSinkTest::CreateFactory()); } }; TEST_CASE_METHOD(OneGlobalSinkFixture, "Global static logger with one sink", "[logger]") { // logs should be forwarded to sink instance: 0 int instance = 0; // create log outside of log limit Logger::Log(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance).empty()); SECTION("create log within log limit") { Logger::Log(LogLevel::Info, "second"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 1); CHECK(prints[0] == "INFO: second"); SECTION("increase log limit create log within log limit") { LogConfig::SetLogLimit(LogLevel::Trace); Logger::Log(LogLevel::Trace, "third"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 2); CHECK(prints[1] == "TRACE: third"); SECTION("log via lambda function") { Logger::Log(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 3); CHECK(prints[2] == "TRACE: forth"); } } } } TEST_CASE_METHOD(OneGlobalSinkFixture, "Local named logger using one global sink", "[logger]") { // create logger with sink instances from global LogConfig Logger logger("TestLogger"); // logs should be forwarded to same sink instance as before: 0 int instance = 0; // create log outside of log limit logger.Emit(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance).empty()); SECTION("create log within log limit") { logger.Emit(LogLevel::Info, "second"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 1); CHECK(prints[0] == "INFO (TestLogger): second"); SECTION("increase log limit create log within log limit") { logger.SetLogLimit(LogLevel::Trace); logger.Emit(LogLevel::Trace, "third"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 2); CHECK(prints[1] == "TRACE (TestLogger): third"); SECTION("log via lambda function") { logger.Emit(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 3); CHECK(prints[2] == "TRACE (TestLogger): forth"); } } } } TEST_CASE_METHOD(OneGlobalSinkFixture, "Local named logger with its own sink instance" "[logger]") { // create logger with separate sink instance Logger logger("OwnSinkLogger", {LogSinkTest::CreateFactory()}); // logs should be forwarded to new sink instance: 1 int instance = 1; // create log outside of log limit logger.Emit(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance).empty()); SECTION("create log within log limit") { logger.Emit(LogLevel::Info, "second"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 1); CHECK(prints[0] == "INFO (OwnSinkLogger): second"); SECTION("increase log limit create log within log limit") { logger.SetLogLimit(LogLevel::Trace); logger.Emit(LogLevel::Trace, "third"); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 2); CHECK(prints[1] == "TRACE (OwnSinkLogger): third"); SECTION("log via lambda function") { logger.Emit(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints = TestPrints::Read(instance); REQUIRE(prints.size() == 3); CHECK(prints[2] == "TRACE (OwnSinkLogger): forth"); } } } } TEST_CASE_METHOD(TwoGlobalSinksFixture, "Global static logger with two sinks", "[logger]") { // logs should be forwarded to sink instances: 0 and 1 int instance1 = 0; int instance2 = 1; // create log outside of log limit Logger::Log(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance1).empty()); CHECK(TestPrints::Read(instance2).empty()); SECTION("create log within log limit") { Logger::Log(LogLevel::Info, "second"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 1); REQUIRE(prints2.size() == 1); CHECK(prints1[0] == "INFO: second"); CHECK(prints2[0] == "INFO: second"); SECTION("increase log limit create log within log limit") { LogConfig::SetLogLimit(LogLevel::Trace); Logger::Log(LogLevel::Trace, "third"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 2); REQUIRE(prints2.size() == 2); CHECK(prints1[1] == "TRACE: third"); CHECK(prints2[1] == "TRACE: third"); SECTION("log via lambda function") { Logger::Log(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 3); REQUIRE(prints2.size() == 3); CHECK(prints1[2] == "TRACE: forth"); CHECK(prints2[2] == "TRACE: forth"); } } } } TEST_CASE_METHOD(TwoGlobalSinksFixture, "Local named logger using two global sinks", "[logger]") { // create logger with sink instances from global LogConfig Logger logger("TestLogger"); // logs should be forwarded to same sink instances: 0 and 1 int instance1 = 0; int instance2 = 1; // create log outside of log limit logger.Emit(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance1).empty()); CHECK(TestPrints::Read(instance2).empty()); SECTION("create log within log limit") { logger.Emit(LogLevel::Info, "second"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 1); REQUIRE(prints2.size() == 1); CHECK(prints1[0] == "INFO (TestLogger): second"); CHECK(prints2[0] == "INFO (TestLogger): second"); SECTION("increase log limit create log within log limit") { logger.SetLogLimit(LogLevel::Trace); logger.Emit(LogLevel::Trace, "third"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 2); REQUIRE(prints2.size() == 2); CHECK(prints1[1] == "TRACE (TestLogger): third"); CHECK(prints2[1] == "TRACE (TestLogger): third"); SECTION("log via lambda function") { logger.Emit(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 3); REQUIRE(prints2.size() == 3); CHECK(prints1[2] == "TRACE (TestLogger): forth"); CHECK(prints2[2] == "TRACE (TestLogger): forth"); } } } } TEST_CASE_METHOD(TwoGlobalSinksFixture, "Local named logger with its own two sink instances", "[logger]") { // create logger with separate sink instances Logger logger("OwnSinkLogger", {LogSinkTest::CreateFactory(), LogSinkTest::CreateFactory()}); // logs should be forwarded to new sink instances: 2 and 3 int instance1 = 2; int instance2 = 3; // create log outside of log limit logger.Emit(LogLevel::Trace, "first"); CHECK(TestPrints::Read(instance1).empty()); CHECK(TestPrints::Read(instance2).empty()); SECTION("create log within log limit") { logger.Emit(LogLevel::Info, "second"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 1); REQUIRE(prints2.size() == 1); CHECK(prints1[0] == "INFO (OwnSinkLogger): second"); CHECK(prints2[0] == "INFO (OwnSinkLogger): second"); SECTION("increase log limit create log within log limit") { logger.SetLogLimit(LogLevel::Trace); logger.Emit(LogLevel::Trace, "third"); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 2); REQUIRE(prints2.size() == 2); CHECK(prints1[1] == "TRACE (OwnSinkLogger): third"); CHECK(prints2[1] == "TRACE (OwnSinkLogger): third"); SECTION("log via lambda function") { logger.Emit(LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints1 = TestPrints::Read(instance1); auto prints2 = TestPrints::Read(instance2); REQUIRE(prints1.size() == 3); REQUIRE(prints2.size() == 3); CHECK(prints1[2] == "TRACE (OwnSinkLogger): forth"); CHECK(prints2[2] == "TRACE (OwnSinkLogger): forth"); } } } } TEST_CASE_METHOD(OneGlobalSinkFixture, "Common interface for global and local named loggers" "[logger]") { // global logs will be forwarded to instance: 0 int global_instance = 0; // create local logger with separate sink instance Logger logger("OwnSinkLogger", {LogSinkTest::CreateFactory()}); // local logs should be forwarded to new sink instance: 1 int local_instance = 1; SECTION("global instance") { // create log outside of log limit Logger::Log(nullptr, LogLevel::Trace, "first"); CHECK(TestPrints::Read(global_instance).empty()); SECTION("create log within log limit") { Logger::Log(nullptr, LogLevel::Info, "second"); auto prints = TestPrints::Read(global_instance); REQUIRE(prints.size() == 1); CHECK(prints[0] == "INFO: second"); SECTION("increase log limit create log within log limit") { LogConfig::SetLogLimit(LogLevel::Trace); Logger::Log(nullptr, LogLevel::Trace, "third"); auto prints = TestPrints::Read(global_instance); REQUIRE(prints.size() == 2); CHECK(prints[1] == "TRACE: third"); SECTION("log via lambda function") { Logger::Log(nullptr, LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints = TestPrints::Read(global_instance); REQUIRE(prints.size() == 3); CHECK(prints[2] == "TRACE: forth"); } } } } SECTION("named instance") { // create log outside of log limit Logger::Log(&logger, LogLevel::Trace, "first"); CHECK(TestPrints::Read(local_instance).empty()); SECTION("create log within log limit") { Logger::Log(&logger, LogLevel::Info, "second"); auto prints = TestPrints::Read(local_instance); REQUIRE(prints.size() == 1); CHECK(prints[0] == "INFO (OwnSinkLogger): second"); SECTION("increase log limit create log within log limit") { logger.SetLogLimit(LogLevel::Trace); Logger::Log(&logger, LogLevel::Trace, "third"); auto prints = TestPrints::Read(local_instance); REQUIRE(prints.size() == 2); CHECK(prints[1] == "TRACE (OwnSinkLogger): third"); SECTION("log via lambda function") { Logger::Log(&logger, LogLevel::Trace, [] { return std::string{"forth"}; }); auto prints = TestPrints::Read(local_instance); REQUIRE(prints.size() == 3); CHECK(prints[2] == "TRACE (OwnSinkLogger): forth"); } } } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/main/000077500000000000000000000000001516554100600235175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/main/TARGETS000066400000000000000000000010361516554100600245530ustar00rootroot00000000000000{ "install_cas": { "type": ["@", "rules", "CC/test", "test"] , "name": ["install_cas"] , "srcs": ["install_cas.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/main", "install_cas"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "main"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["main"] , "deps": ["install_cas"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/main/install_cas.test.cpp000066400000000000000000000073501516554100600275020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/main/install_cas.hpp" #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/crypto/hash_function.hpp" TEST_CASE("ObjectInfoFromLiberalString", "[artifact]") { auto expected = *Artifact::ObjectInfo::FromString( HashFunction::Type::GitSHA1, "[5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:f]"); auto expected_as_tree = *Artifact::ObjectInfo::FromString( HashFunction::Type::GitSHA1, "[5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:0:t]"); // Check (default) file hashes CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "[5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:f]", /*has_remote=*/false) == expected); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:f]", /*has_remote=*/false) == expected); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "[5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:f", /*has_remote=*/false) == expected); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:f", /*has_remote=*/false) == expected); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:file", /*has_remote=*/false) == expected); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:11:notavalidletter", /*has_remote=*/false) == expected); // Without size, which is not honored in equality CHECK( ObjectInfoFromLiberalString(HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689", /*has_remote=*/false) == expected); CHECK( ObjectInfoFromLiberalString(HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:", /*has_remote=*/false) == expected); // Syntactically invalid size should be ignored CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:xyz", /*has_remote=*/false) == expected); // Check tree hashes CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689::t", /*has_remote=*/false) == expected_as_tree); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689::tree", /*has_remote=*/false) == expected_as_tree); CHECK(ObjectInfoFromLiberalString( HashFunction::Type::GitSHA1, "5e1c309dae7f45e0f39b1bf3ac3cd9db12e7d689:xyz:t", /*has_remote=*/false) == expected_as_tree); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/000077500000000000000000000000001516554100600256135ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/TARGETS000066400000000000000000000045001516554100600266460ustar00rootroot00000000000000{ "task": { "type": ["@", "rules", "CC/test", "test"] , "name": ["task"] , "srcs": ["task.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/multithreading", "task"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "multithreading"] } , "task_system": { "type": ["@", "rules", "CC/test", "test"] , "name": ["task_system"] , "srcs": ["task_system.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/utils/cpp", "atomic"] , ["", "catch-main"] , ["utils", "container_matchers"] ] , "stage": ["test", "buildtool", "multithreading"] } , "async_map_node": { "type": ["@", "rules", "CC/test", "test"] , "name": ["async_map_node"] , "srcs": ["async_map_node.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/multithreading", "async_map_node"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["", "catch-main"] , ["utils", "container_matchers"] ] , "stage": ["test", "buildtool", "multithreading"] } , "async_map": { "type": ["@", "rules", "CC/test", "test"] , "name": ["async_map"] , "srcs": ["async_map.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/multithreading", "async_map"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["", "catch-main"] , ["utils", "container_matchers"] ] , "stage": ["test", "buildtool", "multithreading"] } , "async_map_consumer": { "type": ["@", "rules", "CC/test", "test"] , "name": ["async_map_consumer"] , "srcs": ["async_map_consumer.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/multithreading", "async_map_consumer"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["", "catch-main"] , ["utils", "container_matchers"] ] , "stage": ["test", "buildtool", "multithreading"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["multithreading"] , "deps": [ "async_map" , "async_map_consumer" , "async_map_node" , "task" , "task_system" ] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/async_map.test.cpp000066400000000000000000000045641516554100600312600ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/async_map.hpp" #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/multithreading/task_system.hpp" TEST_CASE("Single-threaded: nodes only created once", "[async_map]") { AsyncMap map; auto* key_node = map.GetOrCreateNode("key"); CHECK(key_node != nullptr); auto* other_node = map.GetOrCreateNode("otherkey"); CHECK(other_node != nullptr); auto* should_be_key_node = map.GetOrCreateNode("key"); CHECK(should_be_key_node != nullptr); CHECK(key_node != other_node); CHECK(key_node == should_be_key_node); } TEST_CASE("Nodes only created once and survive the map destruction", "[async_map]") { using NodePtr = typename AsyncMap::NodePtr; NodePtr key_node{nullptr}; NodePtr other_node{nullptr}; NodePtr should_be_key_node{nullptr}; { AsyncMap map; { TaskSystem ts; ts.QueueTask([&key_node, &map]() { auto* node = map.GetOrCreateNode("key"); CHECK(node != nullptr); key_node = node; }); ts.QueueTask([&other_node, &map]() { auto* node = map.GetOrCreateNode("otherkey"); CHECK(node != nullptr); other_node = node; }); ts.QueueTask([&should_be_key_node, &map]() { auto* node = map.GetOrCreateNode("key"); CHECK(node != nullptr); should_be_key_node = node; }); } } CHECK(key_node != nullptr); CHECK(other_node != nullptr); CHECK(should_be_key_node != nullptr); CHECK(key_node != other_node); CHECK(key_node == should_be_key_node); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/async_map_consumer.test.cpp000066400000000000000000000261501516554100600331660ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/async_map_consumer.hpp" #include #include // for fixed width integral types #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "src/buildtool/multithreading/task_system.hpp" auto FibonacciMapConsumer() -> AsyncMapConsumer { auto value_creator = [](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { if (key < 0) { (*logger)("index needs to be non-negative", true); return; } if (key < 2) { (*setter)(uint64_t{static_cast(key)}); return; } (*subcaller)( std::vector{key - 2, key - 1}, [setter](auto const& values) { (*setter)(*values[0] + *values[1]); }, logger); }; return AsyncMapConsumer{value_creator}; } auto FibOnEvenConsumer() -> AsyncMapConsumer { auto value_creator = [](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { if (key < 0) { (*logger)("index needs to be non-negative (and actually even)", true); return; } if (key == 0) { (*setter)(uint64_t{static_cast(0)}); return; } if (key == 2) { (*setter)(uint64_t{static_cast(1)}); return; } (*subcaller)( std::vector{key - 4, key - 2}, [setter](auto const& values) { (*setter)(*values[0] + *values[1]); }, logger); }; return AsyncMapConsumer{value_creator}; } auto CountToMaxConsumer(int max_val, int step = 1, bool cycle = false) -> AsyncMapConsumer { auto value_creator = [max_val, step, cycle](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { if (key < 0 or key > max_val) { // intentional bug: non-fatal abort (*logger)("index out of range", false); return; } if (key == max_val) { // will never be reached if cycle==true (*setter)(uint64_t{static_cast(key)}); return; } auto next = key + step; if (cycle) { next %= max_val; } (*subcaller)( {next}, [setter](auto const& values) { (*setter)(std::uint64_t{*values[0]}); }, logger); }; return AsyncMapConsumer{value_creator}; } TEST_CASE("Fibonacci", "[async_map_consumer]") { std::uint64_t result{}; int const index{92}; bool execution_failed = false; std::uint64_t const expected_result{7540113804746346429}; auto mapconsumer = FibonacciMapConsumer(); { TaskSystem ts; mapconsumer.ConsumeAfterKeysReady( &ts, {index}, [&result](auto const& values) { result = *values[0]; }, [&execution_failed](std::string const& /*unused*/, bool /*unused*/) { execution_failed = true; }); } CHECK(not execution_failed); CHECK(result == expected_result); } TEST_CASE("Values only used once nodes are marked ready", "[async_map_consumer]") { AsyncMapConsumer consume_when_ready{[](auto /*unused*/, auto setter, auto logger, auto subcaller, auto const& key) { if (key == 0) { (*setter)(true); return; } (*subcaller)( {key - 1}, [setter, logger, key](auto const& values) { auto const ready_when_used = values[0]; if (not ready_when_used) { (*logger)(std::to_string(key), true); } (*setter)(true); }, logger); }}; std::vector value_used_before_ready{}; std::mutex vectorm; bool final_value{false}; int const starting_index = 100; { TaskSystem ts; consume_when_ready.ConsumeAfterKeysReady( &ts, {starting_index}, [&final_value](auto const& values) { final_value = values[0]; }, [&value_used_before_ready, &vectorm](std::string const& key, bool /*unused*/) { std::unique_lock l{vectorm}; value_used_before_ready.push_back(key); }); } CHECK(value_used_before_ready.empty()); CHECK(final_value); } TEST_CASE("No subcalling necessary", "[async_map_consumer]") { AsyncMapConsumer identity{ [](auto /*unused*/, auto setter, [[maybe_unused]] auto logger, [[maybe_unused]] auto subcaller, auto const& key) { (*setter)(int{key}); }}; std::vector final_values{}; std::vector const keys{1, 23, 4}; { TaskSystem ts; identity.ConsumeAfterKeysReady( &ts, keys, [&final_values](auto const& values) { std::transform(values.begin(), values.end(), std::back_inserter(final_values), [](auto* val) { return *val; }); }, [](std::string const& /*unused*/, bool /*unused*/) {}); } CHECK(keys == final_values); } TEST_CASE("FibOnEven", "[async_map_consumer]") { std::uint64_t result{}; int const index{184}; bool execution_failed = false; std::uint64_t const expected_result{7540113804746346429}; auto mapconsumer = FibOnEvenConsumer(); { TaskSystem ts; mapconsumer.ConsumeAfterKeysReady( &ts, {index}, [&result](auto const& values) { result = *values[0]; }, [&execution_failed](std::string const& /*unused*/, bool /*unused*/) { execution_failed = true; }); } CHECK(not execution_failed); CHECK(result == expected_result); } TEST_CASE("ErrorPropagation", "[async_map_consumer]") { int const index{183}; // Odd number, will fail bool execution_failed = false; bool consumer_called = false; std::atomic fail_cont_counter{0}; auto mapconsumer = FibOnEvenConsumer(); { TaskSystem ts; mapconsumer.ConsumeAfterKeysReady( &ts, {index}, [&consumer_called](auto const& /*unused*/) { consumer_called = true; }, [&execution_failed](std::string const& /*unused*/, bool /*unused*/) { execution_failed = true; }, [&fail_cont_counter]() { fail_cont_counter++; }); } CHECK(execution_failed); CHECK(not consumer_called); CHECK(fail_cont_counter == 1); } TEST_CASE("Failure detection", "[async_map_consumer]") { int const kMaxVal = 1000; // NOLINT std::optional value{std::nullopt}; bool failed{}; SECTION("Unfinished pending keys") { static constexpr int kStep = 3; REQUIRE(std::lcm(kMaxVal, kStep) > kMaxVal); auto map = CountToMaxConsumer(kMaxVal, kStep); { TaskSystem ts; map.ConsumeAfterKeysReady( &ts, {0}, [&value](auto const& values) { value = *values[0]; }, [&failed](std::string const& /*unused*/, bool fatal) { failed = failed or fatal; }); } CHECK_FALSE(value); CHECK_FALSE(failed); CHECK_FALSE(map.DetectCycle()); auto const pending = map.GetPendingKeys(); CHECK_FALSE(pending.empty()); std::vector expected{}; expected.reserve(kMaxVal + 1); for (int i = 0; i < kMaxVal + kStep; i += kStep) { expected.emplace_back(i); } CHECK_THAT(pending, Catch::Matchers::UnorderedEquals(expected)); } SECTION("Cycle containing all unfinished keys") { auto map = CountToMaxConsumer(kMaxVal, 1, /*cycle=*/true); { TaskSystem ts; map.ConsumeAfterKeysReady( &ts, {0}, [&value](auto const& values) { value = *values[0]; }, [&failed](std::string const& /*unused*/, bool fatal) { failed = failed or fatal; }); } CHECK_FALSE(value); CHECK_FALSE(failed); auto const pending = map.GetPendingKeys(); CHECK_FALSE(pending.empty()); auto const cycle = map.DetectCycle(); REQUIRE(cycle); // pending contains all keys from cycle (except last duplicate key) CHECK_THAT(pending, Catch::Matchers::UnorderedEquals( {cycle->begin(), cycle->end() - 1})); // cycle contains keys in correct order std::vector expected{}; expected.reserve(kMaxVal + 1); for (int i = cycle->at(0); i < cycle->at(0) + kMaxVal + 1; ++i) { expected.emplace_back(i % kMaxVal); } CHECK_THAT(*cycle, Catch::Matchers::Equals(expected)); } SECTION("No cycle and no unfinished keys") { auto map = CountToMaxConsumer(kMaxVal); { TaskSystem ts; map.ConsumeAfterKeysReady( &ts, {0}, [&value](auto const& values) { value = *values[0]; }, [&failed](std::string const& /*unused*/, bool fatal) { failed = failed or fatal; }); } REQUIRE(value); CHECK(*value == kMaxVal); CHECK_FALSE(failed); CHECK_FALSE(map.DetectCycle()); CHECK(map.GetPendingKeys().empty()); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/async_map_node.test.cpp000066400000000000000000000100331516554100600322510ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/async_map_node.hpp" #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "src/buildtool/multithreading/task_system.hpp" TEST_CASE("No task is queued if the node is never ready", "[async_map_node]") { std::vector tasks; std::mutex m; AsyncMapNode node_never_ready{0}; { TaskSystem ts; CHECK_FALSE( node_never_ready.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(0); })); CHECK_FALSE( node_never_ready.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(1); })); CHECK_FALSE( node_never_ready.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(2); })); } CHECK(tasks.empty()); } TEST_CASE("Value is set correctly", "[async_map_node]") { AsyncMapNode node{0}; { TaskSystem ts; node.SetAndQueueAwaitingTasks(&ts, true); } CHECK(node.GetValue()); } TEST_CASE("Tasks are queued correctly", "[async_map_node]") { AsyncMapNode node{0}; std::vector tasks; std::mutex m; { TaskSystem ts; CHECK_FALSE(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(0); })); CHECK_FALSE(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(1); })); CHECK_FALSE(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(2); })); { std::unique_lock l{m}; CHECK(tasks.empty()); } node.SetAndQueueAwaitingTasks(&ts, "ready"); CHECK(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(3); })); CHECK(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(4); })); CHECK(node.AddOrQueueAwaitingTask(&ts, [&tasks, &m]() { std::unique_lock l{m}; // NOLINTNEXTLINE(readability-magic-numbers,cppcoreguidelines-avoid-magic-numbers) tasks.push_back(5); })); } CHECK(node.GetValue() == "ready"); CHECK_THAT( tasks, Catch::Matchers::UnorderedEquals(std::vector{0, 1, 2, 3, 4, 5})); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/task.test.cpp000066400000000000000000000307201516554100600302410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/task.hpp" #include #include // std::move #include "catch2/catch_test_macros.hpp" namespace { constexpr int kDummyValue{5}; struct StatelessCallable { void operator()() noexcept {} }; struct ValueCaptureCallable { explicit ValueCaptureCallable(int i) noexcept : number{i} {} // NOLINTNEXTLINE void operator()() noexcept { number += kDummyValue; } int number; }; struct RefCaptureCallable { // NOLINTNEXTLINE(google-runtime-references) explicit RefCaptureCallable(int& i) noexcept : number{i} {} // NOLINTNEXTLINE void operator()() noexcept { number += 3; } int& number; }; } // namespace TEST_CASE("Default constructed task is empty", "[task]") { Task t; CHECK(not t); CHECK(not(Task())); CHECK(not(Task{})); } TEST_CASE("Task constructed from empty function is empty", "[task]") { std::function empty_function; Task t_from_empty_function{empty_function}; CHECK(not Task(std::function{})); CHECK(not Task(empty_function)); CHECK(not t_from_empty_function); } TEST_CASE("Task constructed from user defined callable object is not empty", "[task]") { SECTION("Stateless struct") { Task t{StatelessCallable{}}; StatelessCallable callable; Task t_from_named_callable{callable}; CHECK(Task{StatelessCallable{}}); CHECK(Task{callable}); CHECK(t); CHECK(t_from_named_callable); } SECTION("Statefull struct") { SECTION("Reference capture") { int a = 2; Task t_ref{RefCaptureCallable{a}}; RefCaptureCallable three_adder{a}; Task t_from_named_callable_ref_capture{three_adder}; CHECK(Task{RefCaptureCallable{a}}); CHECK(Task{three_adder}); CHECK(t_ref); CHECK(t_from_named_callable_ref_capture); } SECTION("Value capture") { Task t_value{ValueCaptureCallable{1}}; ValueCaptureCallable callable{2}; Task t_from_named_callable_value_capture{callable}; CHECK(Task{ValueCaptureCallable{3}}); CHECK(Task{callable}); CHECK(t_value); CHECK(t_from_named_callable_value_capture); } } } TEST_CASE("Task constructed from lambda is not empty", "[task]") { SECTION("Stateless lambda") { Task t{[]() {}}; auto callable = []() {}; Task t_from_named_callable{callable}; CHECK(Task{[]() {}}); CHECK(Task{callable}); CHECK(t); CHECK(t_from_named_callable); } SECTION("Statefull lambda") { SECTION("Reference capture") { int a = 2; Task t_ref{[&a]() { a += 3; }}; auto lambda = [&a]() { a += 3; }; Task t_from_named_lambda_ref_capture{lambda}; CHECK(Task{[&a]() { a += 3; }}); CHECK(Task{lambda}); CHECK(t_ref); CHECK(t_from_named_lambda_ref_capture); } SECTION("Value capture") { int a = 1; // NOLINTNEXTLINE Task t_value{[num = a]() mutable { num += kDummyValue; // get rid of "set but unused var" static_cast(num); }}; // NOLINTNEXTLINE auto lambda = [num = a]() mutable { num += kDummyValue; // get rid of "set but unused var" static_cast(num); }; Task t_from_named_lambda_value_capture{lambda}; CHECK(Task{[num = a]() mutable { num += kDummyValue; // get rid of "set but unused var" static_cast(num); }}); CHECK(Task{lambda}); CHECK(t_value); CHECK(t_from_named_lambda_value_capture); } } } TEST_CASE("Task can be executed and doesn't steal contents", "[task]") { SECTION("User defined object") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; ValueCaptureCallable add_five{num}; Task t_add_five{add_five}; CHECK(add_five.number == initial_value); t_add_five(); // Internal data has been copied once again to the Task, so what is // modified in the call to the task op() is not the data we can // observe from the struct we created (add_five.number) CHECK(add_five.number == initial_value); CHECK(num == initial_value); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; RefCaptureCallable add_three{num}; Task t_add_three{add_three}; CHECK(add_three.number == initial_value); t_add_three(); // In this case, data modified by the task is the same than the one // in the struct, so we can observe the change CHECK(add_three.number == initial_value + 3); CHECK(&num == &add_three.number); } } SECTION("Anonymous lambda function") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE Task t_add_five{[a = num]() mutable { a += kDummyValue; // get rid of "set but unused var" static_cast(a); }}; t_add_five(); // Internal data can not be observed, external data does not change CHECK(num == initial_value); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE Task t_add_three{[&num]() { num += 3; }}; t_add_three(); // Internal data can not be observed, external data changes CHECK(num == initial_value + 3); } } SECTION("Named lambda function") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE auto add_five = [a = num]() mutable { a += kDummyValue; // get rid of "set but unused var" static_cast(a); }; Task t_add_five{add_five}; t_add_five(); // Internal data can not be observed, external data does not change CHECK(num == initial_value); // Lambda can be still called (we can't observe side effects) add_five(); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE auto add_three = [&num]() { num += 3; }; Task t_add_three{add_three}; t_add_three(); // Internal data can not be observed, external data changes CHECK(num == initial_value + 3); // Lambda can be still called (and side effects are as expected) add_three(); CHECK(num == initial_value + 6); } } SECTION("std::function") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE std::function add_five{[a = num]() mutable { a += kDummyValue; // get rid of "set but unused var" static_cast(a); }}; Task t_add_five{add_five}; t_add_five(); // Internal data can not be observed, external data does not change CHECK(num == initial_value); // Original function still valid (side effects not observable) CHECK(add_five); add_five(); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE std::function add_three{[&num]() { num += 3; }}; Task t_add_three{add_three}; t_add_three(); // Internal data can not be observed, external data changes CHECK(num == initial_value + 3); // Original function still valid (and side effects are as expected) CHECK(add_three); add_three(); CHECK(num == initial_value + 6); } } } TEST_CASE("Task moving from named object can be executed", "[task]") { // Constructing Tasks from named objects using Task{std::move(named_object)} // is only a way to explicitly express that the constructor from Task that // will be called will treat `named_object` as an rvalue (temporary object). // We could accomplish the same by using `Task t{Type{args}};` where `Type` // is the type of the callable object. SECTION("User defined object") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; ValueCaptureCallable add_five{num}; // NOLINTNEXTLINE Task t_add_five{std::move(add_five)}; t_add_five(); // No observable side effects CHECK(num == initial_value); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; RefCaptureCallable add_three{num}; // NOLINTNEXTLINE Task t_add_three{std::move(add_three)}; t_add_three(); // External data must have been affected by side effect but in this // case `add_three` is a moved-from object so there is no guarantee // about the data it holds CHECK(num == initial_value + 3); } } // Note that for anonymous lambdas the move constructor of Task is the one // that has already been tested SECTION("Named lambda function") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE auto add_five = [a = num]() mutable { a += kDummyValue; // get rid of "set but unused var" static_cast(a); }; Task t_add_five{std::move(add_five)}; t_add_five(); // Internal data can not be observed, external data does not change CHECK(num == initial_value); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE auto add_three = [&num]() { num += 3; }; Task t_add_three{std::move(add_three)}; t_add_three(); // Internal data can not be observed, external data changes CHECK(num == initial_value + 3); } } SECTION("std::function") { SECTION("Value capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE std::function add_five{[a = num]() mutable { a += kDummyValue; // get rid of "set but unused var" static_cast(a); }}; Task t_add_five{std::move(add_five)}; t_add_five(); // Internal data can not be observed, external data does not change CHECK(num == initial_value); } SECTION("Reference capture") { int const initial_value = 2; int num = initial_value; // NOLINTNEXTLINE std::function add_three{[&num]() { num += 3; }}; Task t_add_three{std::move(add_three)}; t_add_three(); // Internal data can not be observed, external data changes CHECK(num == initial_value + 3); } } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/multithreading/task_system.test.cpp000066400000000000000000000242001516554100600316410ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/multithreading/task_system.hpp" #include #include #include #include #include #include #include #include #include #include // std::iota #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/generators/catch_generators_all.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "src/utils/cpp/atomic.hpp" #include "test/utils/container_matchers.hpp" namespace { enum class CallStatus : std::uint8_t { kNotExecuted, kExecuted }; } // namespace TEST_CASE("Basic", "[task_system]") { SECTION("Empty task system terminates") { { TaskSystem ts; } CHECK(true); } SECTION("0-arguments constructor") { TaskSystem ts; CHECK(ts.NumberOfThreads() == std::thread::hardware_concurrency()); } SECTION("1-argument constructor") { std::size_t const desired_number_of_threads_in_ts = GENERATE(1U, 2U, 5U, 10U, std::thread::hardware_concurrency()); TaskSystem ts(desired_number_of_threads_in_ts); CHECK(ts.NumberOfThreads() == desired_number_of_threads_in_ts); } } TEST_CASE("Side effects of tasks are reflected out of ts", "[task_system]") { SECTION("Lambda function") { auto status = CallStatus::kNotExecuted; { // Make sure that all tasks will be completed before the checks TaskSystem ts; ts.QueueTask([&status]() { status = CallStatus::kExecuted; }); } CHECK(status == CallStatus::kExecuted); } SECTION("std::function") { auto status = CallStatus::kNotExecuted; { TaskSystem ts; std::function f{ [&status]() { status = CallStatus::kExecuted; }}; ts.QueueTask(f); } CHECK(status == CallStatus::kExecuted); } SECTION("Struct") { auto s = CallStatus::kNotExecuted; struct Callable { explicit Callable(CallStatus* cs) : status{cs} {} void operator()() const { *status = CallStatus::kExecuted; } CallStatus* status; }; Callable c{&s}; { TaskSystem ts; ts.QueueTask(c); } CHECK(&s == c.status); CHECK(s == CallStatus::kExecuted); } SECTION("Lambda capturing `this` inside struct") { std::string ext_name{}; struct Wrapper { std::string name; // ts must be second, otherwise name will get destroyed before the // task system is finished. TaskSystem ts; explicit Wrapper(std::string n) : name{std::move(n)} {} void QueueSetAndCheck(std::string* ext) { ts.QueueTask([this, ext]() { SetDefaultName(); CheckDefaultName(ext); }); } void SetDefaultName() { name = "Default"; } void CheckDefaultName(std::string* ext) const { *ext = name; CHECK(name == "Default"); } }; { Wrapper w{"Non-default name"}; w.QueueSetAndCheck(&ext_name); } CHECK(ext_name == "Default"); } } TEST_CASE("All tasks are executed", "[task_system]") { std::size_t const number_of_tasks = 1000; std::vector tasks_executed; std::vector queued_tasks(number_of_tasks); std::iota(std::begin(queued_tasks), std::end(queued_tasks), 0); std::mutex m; { TaskSystem ts; for (auto task_num : queued_tasks) { ts.QueueTask([&tasks_executed, &m, task_num]() { std::unique_lock l{m}; tasks_executed.push_back(task_num); }); } } CHECK_THAT(tasks_executed, HasSameElementsAs>(queued_tasks)); } TEST_CASE("Task is executed even if it needs to wait for a long while", "[task_system]") { auto status = CallStatus::kNotExecuted; // Calculate what would take for the task system to be constructed, queue a // non-sleeping task, execute it and be destructed auto const start_no_sleep = std::chrono::high_resolution_clock::now(); { TaskSystem ts; ts.QueueTask([&status]() { status = CallStatus::kExecuted; }); } auto const end_no_sleep = std::chrono::high_resolution_clock::now(); status = CallStatus::kNotExecuted; std::chrono::nanoseconds const sleep_time = 10 * std::chrono::duration_cast( end_no_sleep - start_no_sleep); auto const start = std::chrono::high_resolution_clock::now(); { TaskSystem ts; ts.QueueTask([&status, sleep_time]() { std::this_thread::sleep_for(sleep_time); status = CallStatus::kExecuted; }); } auto const end = std::chrono::high_resolution_clock::now(); CHECK(end - start > sleep_time); CHECK(status == CallStatus::kExecuted); } TEST_CASE("All threads run until work is done", "[task_system]") { using namespace std::chrono_literals; static auto const kNumThreads = std::thread::hardware_concurrency(); static auto const kFailTimeout = 10s; std::mutex mutex{}; std::condition_variable cv{}; std::unordered_set tids{}; // Add thread id to set and wait for others to do the same. auto store_id = [&tids, &mutex, &cv]() -> void { std::unique_lock lock(mutex); tids.emplace(std::this_thread::get_id()); cv.notify_all(); cv.wait_for( lock, kFailTimeout, [&tids] { return tids.size() == kNumThreads; }); }; SECTION("single task produces multiple tasks") { { TaskSystem ts{kNumThreads}; // Wait some time for all threads to go to sleep. std::this_thread::sleep_for(1s); // All threads should stay alive until their corresponding queue is // filled. One task per thread (assumes round-robin push to queues). for (std::size_t i{}; i < ts.NumberOfThreads(); ++i) { ts.QueueTask([&store_id] { store_id(); }); } } CHECK(tids.size() == kNumThreads); } SECTION("multiple tasks reduce to one, which produces multiple tasks") { atomic counter{}; // All threads wait for counter, last thread creates 'store_id' tasks. auto barrier = [&counter, &store_id](TaskSystem* ts) { auto value = ++counter; if (value == kNumThreads) { counter.notify_all(); // Wait some time for other threads to go to sleep. std::this_thread::sleep_for(1s); // One task per thread (assumes round-robin push to queues). for (std::size_t i{}; i < ts->NumberOfThreads(); ++i) { ts->QueueTask([&store_id] { store_id(); }); } } else { while (value != kNumThreads) { counter.wait(value); value = counter; } } }; { TaskSystem ts{kNumThreads}; // Wait some time for all threads to go to sleep. std::this_thread::sleep_for(1s); // One task per thread (assumes round-robin push to queues). for (std::size_t i{}; i < ts.NumberOfThreads(); ++i) { ts.QueueTask([&barrier, &ts] { barrier(&ts); }); } } CHECK(tids.size() == kNumThreads); } } TEST_CASE("Use finish as system-wide barrier", "[task_system]") { using namespace std::chrono_literals; static auto const kNumThreads = std::thread::hardware_concurrency(); std::vector vec(kNumThreads, 0); std::vector exp0(kNumThreads, 0); std::vector exp1(kNumThreads, 1); std::vector exp2(kNumThreads, 2); { TaskSystem ts{kNumThreads}; // Wait for all threads to go to sleep. ts.Finish(); CHECK(vec == exp0); for (std::size_t i{}; i < ts.NumberOfThreads(); ++i) { ts.QueueTask([&vec, i] { std::this_thread::sleep_for(1s); vec[i] = 1; }); } ts.Finish(); CHECK(vec == exp1); for (std::size_t i{}; i < ts.NumberOfThreads(); ++i) { ts.QueueTask([&vec, i] { std::this_thread::sleep_for(1s); vec[i] = 2; }); } } CHECK(vec == exp2); } TEST_CASE("Shut down a running task system", "[task_system]") { using namespace std::chrono_literals; static auto const kNumThreads = std::thread::hardware_concurrency(); std::atomic count{0}; std::atomic finished{false}; std::function sleeper{}; { TaskSystem ts{kNumThreads}; // sleeper, recursively runs forever sleeper = [&count, &ts, &sleeper]() { ++count; std::this_thread::sleep_for(1s); ts.QueueTask(sleeper); }; // waiter, asynchronous task waiting for task system to finish std::thread waiter{[&finished, &ts] { ts.Finish(); finished = true; }}; // run sleeper ts.QueueTask(sleeper); std::this_thread::sleep_for(1s); // initiate shutdown and join with waiter ts.Shutdown(); waiter.join(); } CHECK(count > 0); CHECK(finished); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/serve_api/000077500000000000000000000000001516554100600245505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/serve_api/TARGETS000066400000000000000000000023701516554100600256060ustar00rootroot00000000000000{ "source_tree_client": { "type": ["utils/serve_service", "CC test"] , "name": ["source_tree_client"] , "srcs": ["source_tree_client.test.cpp"] , "data": [["buildtool/file_system", "test_data"]] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/auth", "auth"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/execution_api/remote", "context"] , ["@", "src", "src/buildtool/file_system", "git_types"] , ["@", "src", "src/buildtool/serve_api/remote", "config"] , ["@", "src", "src/buildtool/serve_api/remote", "source_tree_client"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-serve"] , ["utils", "test_hash_function_type"] , ["utils", "test_serve_config"] ] , "stage": ["test", "buildtool", "serve_api"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["serve_api"] , "deps": ["source_tree_client"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/serve_api/source_tree_client.test.cpp000066400000000000000000000106261516554100600321140ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/serve_api/remote/source_tree_client.hpp" #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" #include "test/utils/serve_service/test_serve_config.hpp" auto const kRootCommit = std::string{"e4fc610c60716286b98cf51ad0c8f0d50f3aebb5"}; auto const kRootId = std::string{"c610db170fbcad5f2d66fe19972495923f3b2536"}; auto const kBazId = std::string{"27b32561185c2825150893774953906c6daa6798"}; auto const kRootSymCommit = std::string{"3ecce3f5b19ad7941c6354d65d841590662f33ef"}; auto const kRootSymId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kBazSymId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"}; TEST_CASE("Serve service client: tree-of-commit request", "[serve_api]") { auto const config = TestServeConfig::ReadFromEnvironment(); REQUIRE(config); REQUIRE(config->remote_address); auto const hash_function = HashFunction{TestHashType::ReadFromEnvironment()}; // Create TLC client Auth auth{}; RetryConfig retry_config{}; RemoteExecutionConfig exec_config{}; RemoteContext const remote_context{.auth = &auth, .retry_config = &retry_config, .exec_config = &exec_config}; SourceTreeClient st_client( *config->remote_address, hash_function, &remote_context); SECTION("Commit in bare checkout") { auto root_id = st_client.ServeCommitTree(kRootCommit, ".", false); REQUIRE(root_id); CHECK_FALSE(root_id->digest); // digest is not provided if not syncing if (ProtocolTraits::IsNative(hash_function.GetType())) { CHECK(root_id->tree == kRootId); } auto baz_id = st_client.ServeCommitTree(kRootCommit, "baz", false); REQUIRE(baz_id); CHECK_FALSE(baz_id->digest); // digest is not provided if not syncing if (ProtocolTraits::IsNative(hash_function.GetType())) { CHECK(baz_id->tree == kBazId); } } SECTION("Commit in non-bare checkout") { auto root_id = st_client.ServeCommitTree(kRootSymCommit, ".", false); REQUIRE(root_id); CHECK_FALSE(root_id->digest); // digest is not provided if not syncing if (ProtocolTraits::IsNative(hash_function.GetType())) { CHECK(root_id->tree == kRootSymId); } auto baz_id = st_client.ServeCommitTree(kRootSymCommit, "baz", false); REQUIRE(baz_id); CHECK_FALSE(baz_id->digest); // digest is not provided if not syncing if (ProtocolTraits::IsNative(hash_function.GetType())) { CHECK(baz_id->tree == kBazSymId); } } SECTION("Subdir not found") { auto root_id = st_client.ServeCommitTree(kRootCommit, "does_not_exist", false); REQUIRE_FALSE(root_id); CHECK(root_id.error() == GitLookupError::Fatal); // fatal failure } SECTION("Commit not known") { auto root_id = st_client.ServeCommitTree( "0123456789abcdef0123456789abcdef01234567", ".", false); REQUIRE_FALSE(root_id); CHECK(root_id.error() == GitLookupError::NotFound); // non-fatal failure } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/000077500000000000000000000000001516554100600242375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/TARGETS000066400000000000000000000055771516554100600253110ustar00rootroot00000000000000{ "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": [ "data/executable_file" , "data/non_executable_file" , "data/subdir1/file1" , "data/subdir1/subdir2/file2" ] , "stage": ["test", "buildtool", "storage"] } , "local_cas": { "type": ["@", "rules", "CC/test", "test"] , "name": ["local_cas"] , "srcs": ["local_cas.test.cpp"] , "data": [["buildtool/storage", "test_data"]] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "storage"] } , "local_ac": { "type": ["@", "rules", "CC/test", "test"] , "name": ["local_ac"] , "srcs": ["local_ac.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "bazel_types"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["", "catch-main"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "storage"] } , "large_object_cas": { "type": ["@", "rules", "CC/test", "test"] , "name": ["large_object_cas"] , "srcs": ["large_object_cas.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/bazel_msg" , "bazel_msg_factory" ] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/common", "ids"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "garbage_collector"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "large_object_utils"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "storage"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["storage"] , "deps": ["large_object_cas", "local_ac", "local_cas"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/000077500000000000000000000000001516554100600251505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/executable_file000077500000000000000000000000041516554100600302100ustar00rootroot00000000000000testjust-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/non_executable_file000077500000000000000000000000041516554100600310620ustar00rootroot00000000000000testjust-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/subdir1/000077500000000000000000000000001516554100600265215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/subdir1/file1000066400000000000000000000000041516554100600274360ustar00rootroot00000000000000testjust-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/subdir1/subdir2/000077500000000000000000000000001516554100600300735ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/data/subdir1/subdir2/file2000066400000000000000000000000041516554100600310110ustar00rootroot00000000000000testjust-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/large_object_cas.test.cpp000066400000000000000000000741261516554100600312010ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/large_object_cas.hpp" #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/large_objects/large_object_utils.hpp" namespace { namespace LargeTestUtils { template class Blob final { public: static constexpr auto kLargeId = "bl_8Mb"; static constexpr std::uintmax_t kLargeSize = 8UL * 1024 * 1024; static constexpr auto kSmallId = "bl_1kB"; static constexpr std::uintmax_t kSmallSize = 1024; static constexpr auto kEmptyId = "bl_0"; static constexpr std::uintmax_t kEmptySize = 0; [[nodiscard]] static auto Create( LocalCAS const& cas, std::string const& id, std::uintmax_t size) noexcept -> std::optional>; [[nodiscard]] static auto Generate(std::string const& id, std::uintmax_t size) noexcept -> std::optional; }; using File = Blob; class Tree final { public: static constexpr auto kLargeId = "tree_4096"; static constexpr std::uintmax_t kLargeSize = 4096; static constexpr auto kSmallId = "tree_1"; static constexpr std::uintmax_t kSmallSize = 1; static constexpr auto kEmptyId = "tree_0"; static constexpr std::uintmax_t kEmptySize = 0; [[nodiscard]] static auto Create( LocalCAS const& cas, std::string const& id, std::uintmax_t entries_count) noexcept -> std::optional>; [[nodiscard]] static auto Generate(std::string const& id, std::uintmax_t entries_count) noexcept -> std::optional; [[nodiscard]] static auto StoreRaw( LocalCAS const& cas, std::filesystem::path const& directory) noexcept -> std::optional; }; } // namespace LargeTestUtils } // namespace // Test splitting of a small tree. TEST_CASE("LargeObjectCAS: split a small tree", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& cas = storage.CAS(); // Create a small tree: using LargeTestUtils::Tree; auto small = Tree::Create(cas, Tree::kSmallId, Tree::kSmallSize); REQUIRE(small); auto const& [digest, path] = *small; // Split must be successful: auto split_pack = cas.SplitTree(digest); REQUIRE(split_pack); // The result must contain one blob digest: CHECK(split_pack->size() == 1); CHECK_FALSE(split_pack->front().IsTree()); } // Test splitting of a large object. The split must be successful and the entry // must be placed to the LargeCAS. The second split of the same object must load // the result from the LargeCAS, no actual split must occur. // The object can be implicitly reconstructed from the LargeCAS. template static void TestLarge(StorageConfig const& storage_config, Storage const& storage) noexcept { SECTION("Large") { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); using TestType = std::conditional_t>; auto const& cas = storage.CAS(); // Create a large object: auto object = TestType::Create(cas, TestType::kLargeId, TestType::kLargeSize); CHECK(object); auto const& [digest, path] = *object; // Split the large object: auto pack_1 = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); CHECK(pack_1); CHECK(pack_1->size() > 1); CHECK(FileSystemManager::RemoveFile(path)); CHECK_FALSE(FileSystemManager::IsFile(path)); SECTION("Split short-circuiting") { // Check the second call loads the entry from the large CAS: auto pack_2 = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); CHECK(pack_2); CHECK(pack_2->size() == pack_1->size()); // There must be no spliced file: CHECK_FALSE(FileSystemManager::IsFile(path)); } SECTION("Splice") { // Check implicit splice: auto spliced_path = kIsTree ? cas.TreePath(digest) : cas.BlobPath(digest, kIsExec); REQUIRE(spliced_path); // The result must be in the same location: CHECK(*spliced_path == path); } SECTION("Uplinking") { // Increment generation: CHECK(GarbageCollector::TriggerGarbageCollection(storage_config)); // Check implicit splice: auto spliced_path = kIsTree ? cas.TreePath(digest) : cas.BlobPath(digest, kIsExec); REQUIRE(spliced_path); // The result must be spliced to the same location: CHECK(*spliced_path == path); // Check the large entry was uplinked too: // Remove the spliced result: CHECK(FileSystemManager::RemoveFile(path)); CHECK_FALSE(FileSystemManager::IsFile(path)); // Call split with disabled uplinking: auto const youngest_storage = ::Generation::Create(&storage_config); auto pack_3 = kIsTree ? youngest_storage.CAS().SplitTree(digest) : youngest_storage.CAS().SplitBlob(digest); REQUIRE(pack_3); CHECK(pack_3->size() == pack_1->size()); // Check there are no spliced results in all generations: for (std::size_t i = 0; i < storage_config.num_generations; ++i) { auto const storage = ::Generation::Create(&storage_config); auto generation_path = kIsTree ? storage.CAS().TreePath(digest) : storage.CAS().BlobPath(digest, kIsExec); REQUIRE_FALSE(generation_path); } } } } // Test splitting of a small object. The split must be successful, but the entry // must not be placed to the LargeCAS. The result of spliting must contain one // blob. // The object cannot be implicitly reconstructed. template static void TestSmall(Storage const& storage) noexcept { SECTION("Small") { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); using TestType = std::conditional_t>; auto const& cas = storage.CAS(); // Create a small object: auto object = TestType::Create(cas, TestType::kSmallId, TestType::kSmallSize); CHECK(object); auto const& [digest, path] = *object; // Split the small object: auto pack_1 = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); CHECK(pack_1); CHECK(pack_1->size() == 1); CHECK_FALSE(pack_1->front().IsTree()); // Test that there is no large entry in the storage: // To ensure there is no split of the initial object, it is removed: CHECK(FileSystemManager::RemoveFile(path)); CHECK_FALSE(FileSystemManager::IsFile(path)); // The part of a small executable is the same file but without the // execution permission. It must be deleted too. if constexpr (kIsExec) { auto part_path = cas.BlobPath(pack_1->front(), false); CHECK(part_path); CHECK(FileSystemManager::RemoveFile(*part_path)); } // Split must not find the large entry: auto pack_2 = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); CHECK_FALSE(pack_2); CHECK(pack_2.error().Code() == LargeObjectErrorCode::FileNotFound); // There must be no spliced file: CHECK_FALSE(FileSystemManager::IsFile(path)); // Check implicit splice fails: auto spliced_path = kIsTree ? cas.TreePath(digest) : cas.BlobPath(digest, kIsExec); CHECK_FALSE(spliced_path); } } // Test splitting of an empty object. The split must be successful, but the // entry must not be placed to the LargeCAS. The result of splitting must be // empty. // The object cannot be implicitly reconstructed. template static void TestEmpty(Storage const& storage) noexcept { SECTION("Empty") { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); using TestType = std::conditional_t>; // Create an empty file: auto temp_path = LargeTestUtils::Blob::Generate( TestType::kEmptyId, TestType::kEmptySize); REQUIRE(temp_path); auto const& cas = storage.CAS(); auto digest = kIsTree ? cas.StoreTree(*temp_path) : cas.StoreBlob(*temp_path, kIsExec); REQUIRE(digest); auto path = kIsTree ? cas.TreePath(*digest) : cas.BlobPath(*digest, kIsExec); REQUIRE(path); // Split the empty object: auto pack_1 = kIsTree ? cas.SplitTree(*digest) : cas.SplitBlob(*digest); CHECK(pack_1); CHECK(pack_1->empty()); // Test that there is no large entry in the storage: // To ensure there is no split of the initial object, it is removed: CHECK(FileSystemManager::RemoveFile(*path)); CHECK_FALSE(FileSystemManager::IsFile(*path)); // Split must not find the large entry: auto pack_2 = kIsTree ? cas.SplitTree(*digest) : cas.SplitBlob(*digest); CHECK_FALSE(pack_2); CHECK(pack_2.error().Code() == LargeObjectErrorCode::FileNotFound); // There must be no spliced file: CHECK_FALSE(FileSystemManager::IsFile(*path)); // Check implicit splice fails: auto spliced_path = kIsTree ? cas.TreePath(*digest) : cas.BlobPath(*digest, kIsExec); CHECK_FALSE(spliced_path); } } // Test splicing from an external source. // 1. The object can be explicitly spliced, if the parts are presented in the // storage. // 2. Explicit splice fails, it the result of splicing is different from // what was expected. // 3. Explicit splice fails, if some parts of the tree are missing. template static void TestExternal(StorageConfig const& storage_config, Storage const& storage) noexcept { SECTION("External") { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); using TestType = std::conditional_t>; auto const& cas = storage.CAS(); // Create a large object: auto object = TestType::Create(cas, TestType::kLargeId, TestType::kLargeSize); CHECK(object); auto const& [digest, path] = *object; // Split the object: auto pack_1 = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); CHECK(pack_1); CHECK(pack_1->size() > 1); // External source is emulated by moving the large entry to an older // generation and promoting the parts of the entry to the youngest // generation: REQUIRE(GarbageCollector::TriggerGarbageCollection(storage_config)); for (auto const& part : *pack_1) { static constexpr bool kIsExecutable = false; REQUIRE(cas.BlobPath(part, kIsExecutable)); } auto const youngest = ::Generation::Create(&storage_config); SECTION("Proper request") { if constexpr (kIsTree) { // Promote the parts of the tree: auto splice = cas.TreePath(digest); REQUIRE(splice); REQUIRE(FileSystemManager::RemoveFile(*splice)); } REQUIRE_FALSE(FileSystemManager::IsFile(path)); // Reconstruct the result from parts: std::ignore = kIsTree ? youngest.CAS().SpliceTree(digest, *pack_1) : youngest.CAS().SpliceBlob(digest, *pack_1, kIsExec); CHECK(FileSystemManager::IsFile(path)); } // Simulate a situation when parts result to an existing file, but it is // not the expected result: SECTION("Digest consistency fail") { // Splice the result to check it will not be affected: auto implicit_splice = kIsTree ? cas.TreePath(digest) : cas.BlobPath(digest, kIsExec); REQUIRE(implicit_splice); REQUIRE(*implicit_splice == path); // Randomize one more object to simulate invalidation: auto small = TestType::Create(cas, TestType::kSmallId, TestType::kSmallSize); REQUIRE(small); auto const& [small_digest, small_path] = *small; // The entry itself is not important, only it's digest is needed: REQUIRE(FileSystemManager::RemoveFile(small_path)); REQUIRE_FALSE(FileSystemManager::IsFile(small_path)); // Invalidation is simulated by reconstructing the small_digest // object from the parts of the initial object: auto splice = kIsTree ? youngest.CAS().SpliceTree(small_digest, *pack_1) : youngest.CAS().SpliceBlob(small_digest, *pack_1, kIsExec); REQUIRE_FALSE(splice); CHECK(splice.error().Code() == LargeObjectErrorCode::InvalidResult); // The initial entry must not be affected: REQUIRE(FileSystemManager::IsFile(path)); } if (kIsTree and ProtocolTraits::IsTreeAllowed( storage_config.hash_function.GetType())) { // Tree invariants check is omitted in compatible mode. SECTION("Tree invariants check fails") { // Check splice fails due to the tree invariants check. auto splice = youngest.CAS().SpliceTree(digest, *pack_1); REQUIRE_FALSE(splice); CHECK(splice.error().Code() == LargeObjectErrorCode::InvalidTree); } } } } // Test compactification of a storage generation. // If there are objects in the storage that have an entry in // the large CAS, they must be deleted during compactification. // All splitable objects in the generation must be split. template static void TestCompactification(StorageConfig const& storage_config, Storage const& storage) { SECTION("Compactify") { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); using TestType = std::conditional_t>; auto const& cas = storage.CAS(); // Create a large object and split it: auto object = TestType::Create(cas, TestType::kLargeId, TestType::kLargeSize); REQUIRE(object); auto& [digest, path] = *object; auto result = kIsTree ? cas.SplitTree(digest) : cas.SplitBlob(digest); REQUIRE(result); // For trees the size must be increased to exceed the internal // compactification threshold: static constexpr auto kExceedThresholdSize = kIsTree ? TestType::kLargeSize * 8 : TestType::kLargeSize; // Create a large object that is to be split during compactification: auto object_2 = TestType::Create( cas, std::string(TestType::kLargeId) + "_2", kExceedThresholdSize); REQUIRE(object_2); auto& [digest_2, path_2] = *object_2; // After an interruption of a build process intermediate unique files // may be present in the storage. To ensure compactification deals with // them properly, a "unique" file is created: auto invalid_object = TestType::Create( cas, std::string(TestType::kLargeId) + "_3", kExceedThresholdSize); REQUIRE(invalid_object); auto& [invalid_digest, invalid_path] = *invalid_object; auto unique_path = CreateUniquePath(invalid_path); REQUIRE(unique_path); REQUIRE(FileSystemManager::Rename(invalid_path, *unique_path)); // Ensure all entries are in the storage: auto get_path = [](auto const& cas, ArtifactDigest const& digest) { return kIsTree ? cas.TreePath(digest) : cas.BlobPath(digest, kIsExec); }; auto const latest = ::Generation::Create(&storage_config); REQUIRE(get_path(latest.CAS(), digest).has_value()); REQUIRE(get_path(latest.CAS(), digest_2).has_value()); REQUIRE(FileSystemManager::IsFile(*unique_path)); // Compactify the youngest generation: // Generation rotation is disabled to exclude uplinking. static constexpr bool kNoRotation = true; REQUIRE(GarbageCollector::TriggerGarbageCollection(storage_config, kNoRotation)); // All entries must be deleted during compactification, and for blobs // and executables there are no synchronized entries in the storage: REQUIRE_FALSE(get_path(latest.CAS(), digest).has_value()); REQUIRE_FALSE(get_path(latest.CAS(), digest_2).has_value()); REQUIRE_FALSE(FileSystemManager::IsFile(*unique_path)); // All valid entries must be implicitly spliceable: REQUIRE(get_path(cas, digest).has_value()); REQUIRE(get_path(cas, digest_2).has_value()); } } TEST_CASE("LocalCAS: Split-Splice", "[storage]") { auto const config = TestStorageConfig::Create(); auto const storage = Storage::Create(&config.Get()); SECTION("File") { TestLarge(config.Get(), storage); TestSmall(storage); TestEmpty(storage); TestExternal(config.Get(), storage); TestCompactification(config.Get(), storage); } SECTION("Tree") { TestLarge(config.Get(), storage); TestSmall(storage); TestEmpty(storage); TestExternal(config.Get(), storage); TestCompactification(config.Get(), storage); } SECTION("Executable") { TestLarge(config.Get(), storage); TestSmall(storage); TestEmpty(storage); TestExternal(config.Get(), storage); TestCompactification(config.Get(), storage); } } TEST_CASE("Skip compactification", "[storage]") { auto const config = TestStorageConfig::Create(); auto const storage = Storage::Create(&config.Get()); auto const& cas = storage.CAS(); // Create a large object in CAS: using LargeTestUtils::File; auto object = File::Create(cas, File::kLargeId, File::kLargeSize); REQUIRE(object.has_value()); auto& [digest, path] = *object; REQUIRE(cas.BlobPath(digest, /*is_executable=*/false).has_value()); // Trigger garbage collection with --all REQUIRE(GarbageCollector::TriggerGarbageCollection( config.Get(), /*no_rotation=*/false, /*gc_all=*/true)); // Check no generation remains: for (std::size_t i = 0; i < config.Get().num_generations; ++i) { CHECK_FALSE( FileSystemManager::Exists(config.Get().GenerationCacheRoot(i))); } } // Test uplinking of nested large objects: // A large tree depends on a number of nested objects: // // large_tree // | - nested_blob // | - nested_tree // | |- other nested entries // | - other entries // // All large entries are preliminarily split and the spliced results are // deleted. The youngest generation is empty. Uplinking must restore the // object(and it's parts) and uplink them properly. TEST_CASE("LargeObjectCAS: uplink nested large objects", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& cas = storage.CAS(); // Randomize a large directory: auto tree_path = LargeTestUtils::Tree::Generate( "nested_tree", LargeTestUtils::Tree::kLargeSize); REQUIRE(tree_path); // Randomize a large nested tree: auto const nested_tree = (*tree_path) / "nested_tree"; REQUIRE(LargeObjectUtils::GenerateDirectory( nested_tree, LargeTestUtils::Tree::kLargeSize)); // Randomize a large nested blob: auto nested_blob = (*tree_path) / "nested_blob"; REQUIRE(LargeObjectUtils::GenerateFile(nested_blob, LargeTestUtils::File::kLargeSize)); // Add the nested tree to the CAS: auto nested_tree_digest = LargeTestUtils::Tree::StoreRaw(cas, nested_tree); REQUIRE(nested_tree_digest); auto nested_tree_path = cas.TreePath(*nested_tree_digest); REQUIRE(nested_tree_path); // Add the nested blob to the CAS: auto nested_blob_digest = cas.StoreBlob(nested_blob, false); REQUIRE(nested_blob_digest); auto nested_blob_path = cas.BlobPath(*nested_blob_digest, false); REQUIRE(nested_blob_path); // Add the initial large directory to the CAS: auto large_tree_digest = LargeTestUtils::Tree::StoreRaw(cas, *tree_path); REQUIRE(large_tree_digest); auto large_tree_path = cas.TreePath(*large_tree_digest); REQUIRE(large_tree_path); // Split large entries: auto split_nested_tree = cas.SplitTree(*nested_tree_digest); REQUIRE(split_nested_tree); auto split_nested_blob = cas.SplitBlob(*nested_blob_digest); REQUIRE(split_nested_blob); auto split_large_tree = cas.SplitTree(*large_tree_digest); REQUIRE(split_large_tree); // Remove the spliced results: REQUIRE(FileSystemManager::RemoveFile(*nested_tree_path)); REQUIRE(FileSystemManager::RemoveFile(*nested_blob_path)); REQUIRE(FileSystemManager::RemoveFile(*large_tree_path)); // Rotate generations: REQUIRE(GarbageCollector::TriggerGarbageCollection(storage_config.Get())); // Ask to splice the large tree: auto result_path = cas.TreePath(*large_tree_digest); REQUIRE(result_path); // Only the main object must be reconstructed: CHECK(FileSystemManager::IsFile(*large_tree_path)); // It's parts must not be reconstructed by default: CHECK_FALSE(FileSystemManager::IsFile(*nested_tree_path)); CHECK_FALSE(FileSystemManager::IsFile(*nested_blob_path)); auto const latest = ::Generation::Create(&storage_config.Get()); // However, in native mode they might be reconstructed on request because // their entries are in the latest generation: if (ProtocolTraits::IsNative( storage_config.Get().hash_function.GetType())) { auto split_nested_tree_2 = latest.CAS().SplitTree(*nested_tree_digest); REQUIRE(split_nested_tree_2); auto split_nested_blob_2 = latest.CAS().SplitBlob(*nested_blob_digest); REQUIRE(split_nested_blob_2); } // Check there are no spliced results in old generations: for (std::size_t i = 1; i < storage_config.Get().num_generations; ++i) { auto const storage = ::Generation::Create(&storage_config.Get(), /*generation=*/i); auto const& generation_cas = storage.CAS(); REQUIRE_FALSE(generation_cas.TreePath(*nested_tree_digest)); REQUIRE_FALSE(generation_cas.TreePath(*large_tree_digest)); REQUIRE_FALSE(generation_cas.BlobPath(*nested_blob_digest, /*is_executable=*/false)); } } namespace { /// \brief Extends the lifetime of large files for the whole set of tests. class TestFilesDirectory final { public: [[nodiscard]] static auto Instance() noexcept -> TestFilesDirectory const& { static TestFilesDirectory directory; return directory; } [[nodiscard]] auto GetPath() const noexcept -> std::filesystem::path { return temp_directory_->GetPath(); } private: TmpDir::Ptr temp_directory_; explicit TestFilesDirectory() noexcept { auto test_dir = FileSystemManager::GetCurrentDirectory() / "tmp"; temp_directory_ = TmpDir::Create(test_dir / "tmp_space"); } }; namespace LargeTestUtils { template auto Blob::Create(LocalCAS const& cas, std::string const& id, std::uintmax_t size) noexcept -> std::optional> { std::optional path; while (not path.has_value()) { path = Generate(id, size); auto digest = path.has_value() ? ArtifactDigestFactory::HashFileAs( cas.GetHashFunction(), *path) : std::nullopt; if (not digest) { return std::nullopt; } if (cas.BlobPath(digest.value(), kIsExecutable).has_value()) { if (not FileSystemManager::RemoveFile(*path)) { return std::nullopt; } path.reset(); } } auto digest = path ? cas.StoreBlob(*path, kIsExecutable) : std::nullopt; auto blob_path = digest ? cas.BlobPath(*digest, kIsExecutable) : std::nullopt; if (digest and blob_path) { return std::make_pair(std::move(*digest), std::move(*blob_path)); } return std::nullopt; } template auto Blob::Generate(std::string const& id, std::uintmax_t size) noexcept -> std::optional { std::string const path_id = "blob" + id; auto path = TestFilesDirectory::Instance().GetPath() / path_id; if (FileSystemManager::IsFile(path) or LargeObjectUtils::GenerateFile(path, size)) { return path; } return std::nullopt; } auto Tree::Create(LocalCAS const& cas, std::string const& id, std::uintmax_t entries_count) noexcept -> std::optional> { auto path = Generate(id, entries_count); auto digest = path ? StoreRaw(cas, *path) : std::nullopt; auto cas_path = digest ? cas.TreePath(*digest) : std::nullopt; if (digest and cas_path) { return std::make_pair(std::move(*digest), std::move(*cas_path)); } return std::nullopt; } auto Tree::Generate(std::string const& id, std::uintmax_t entries_count) noexcept -> std::optional { std::string const path_id = "tree" + id; auto path = TestFilesDirectory::Instance().GetPath() / path_id; if (FileSystemManager::IsDirectory(path) or LargeObjectUtils::GenerateDirectory(path, entries_count)) { return path; } return std::nullopt; } auto Tree::StoreRaw(LocalCAS const& cas, std::filesystem::path const& directory) noexcept -> std::optional { if (not FileSystemManager::IsDirectory(directory)) { return std::nullopt; } auto store_blob = [&cas](std::filesystem::path const& path, auto is_exec) -> std::optional { return cas.StoreBlob(path, is_exec); }; auto store_tree = [&cas](std::string const& content) -> std::optional { return cas.StoreTree(content); }; auto store_symlink = [&cas](std::string const& content) -> std::optional { return cas.StoreBlob(content); }; return ProtocolTraits::IsNative(cas.GetHashFunction().GetType()) ? BazelMsgFactory::CreateGitTreeDigestFromLocalTree( directory, store_blob, store_tree, store_symlink) : BazelMsgFactory::CreateDirectoryDigestFromLocalTree( directory, store_blob, store_tree, store_symlink); } } // namespace LargeTestUtils } // namespace just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/local_ac.test.cpp000066400000000000000000000141501516554100600274570ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/storage/local_ac.hpp" #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" [[nodiscard]] static auto RunDummyExecution( gsl::not_null const*> const& ac, gsl::not_null const*> const& cas_, ArtifactDigest const& action_id, std::string const& seed) -> bool; TEST_CASE("LocalAC: Single action, single result", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& ac = storage.ActionCache(); auto const& cas = storage.CAS(); auto action_id = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "action"); CHECK(not ac.CachedResult(action_id)); CHECK(RunDummyExecution(&ac, &cas, action_id, "result")); auto ac_result = ac.CachedResult(action_id); CHECK(ac_result); } TEST_CASE("LocalAC: Two different actions, two different results", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& ac = storage.ActionCache(); auto const& cas = storage.CAS(); auto action_id1 = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "action1"); auto action_id2 = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "action2"); CHECK(not ac.CachedResult(action_id1)); CHECK(not ac.CachedResult(action_id2)); std::string result_content1{}; std::string result_content2{}; CHECK(RunDummyExecution(&ac, &cas, action_id1, "result1")); auto ac_result1 = ac.CachedResult(action_id1); REQUIRE(ac_result1); CHECK(ac_result1->SerializeToString(&result_content1)); CHECK(RunDummyExecution(&ac, &cas, action_id2, "result2")); auto ac_result2 = ac.CachedResult(action_id2); REQUIRE(ac_result2); CHECK(ac_result2->SerializeToString(&result_content2)); // check different actions, different result CHECK(action_id1.hash() != action_id2.hash()); CHECK(result_content1 != result_content2); } TEST_CASE("LocalAC: Two different actions, same two results", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& ac = storage.ActionCache(); auto const& cas = storage.CAS(); auto action_id1 = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "action1"); auto action_id2 = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "action2"); CHECK(not ac.CachedResult(action_id1)); CHECK(not ac.CachedResult(action_id2)); std::string result_content1{}; std::string result_content2{}; CHECK(RunDummyExecution(&ac, &cas, action_id1, "same result")); auto ac_result1 = ac.CachedResult(action_id1); REQUIRE(ac_result1); CHECK(ac_result1->SerializeToString(&result_content1)); CHECK(RunDummyExecution(&ac, &cas, action_id2, "same result")); auto ac_result2 = ac.CachedResult(action_id2); REQUIRE(ac_result2); CHECK(ac_result2->SerializeToString(&result_content2)); // check different actions, but same result CHECK(action_id1.hash() != action_id2.hash()); CHECK(result_content1 == result_content2); } TEST_CASE("LocalAC: Same two actions, two different results", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& ac = storage.ActionCache(); auto const& cas = storage.CAS(); auto action_id = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, "same action"); CHECK(not ac.CachedResult(action_id)); std::string result_content1{}; std::string result_content2{}; CHECK(RunDummyExecution(&ac, &cas, action_id, "result1")); auto ac_result1 = ac.CachedResult(action_id); REQUIRE(ac_result1); CHECK(ac_result1->SerializeToString(&result_content1)); CHECK(RunDummyExecution(&ac, &cas, action_id, "result2")); // updated auto ac_result2 = ac.CachedResult(action_id); REQUIRE(ac_result2); CHECK(ac_result2->SerializeToString(&result_content2)); // check same actions, different cached result CHECK(result_content1 != result_content2); } auto RunDummyExecution(gsl::not_null const*> const& ac, gsl::not_null const*> const& cas_, ArtifactDigest const& action_id, std::string const& seed) -> bool { bazel_re::ActionResult result{}; *result.add_output_files() = [&]() { bazel_re::OutputFile out{}; out.set_path(seed); auto digest = cas_->StoreBlob(""); *out.mutable_digest() = ArtifactDigestFactory::ToBazel(*digest); out.set_is_executable(false); return out; }(); return ac->StoreResult(action_id, result); } just-buildsystem-justbuild-b1fb5fa/test/buildtool/storage/local_cas.test.cpp000066400000000000000000000152501516554100600276440ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" TEST_CASE("LocalCAS: Add blob to storage from bytes", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& cas = storage.CAS(); std::string test_bytes("test"); auto test_digest = ArtifactDigestFactory::HashDataAs( storage_config.Get().hash_function, test_bytes); // check blob not in storage CHECK(not cas.BlobPath(test_digest, true)); CHECK(not cas.BlobPath(test_digest, false)); // ensure previous calls did not accidentially create the blob CHECK(not cas.BlobPath(test_digest, true)); CHECK(not cas.BlobPath(test_digest, false)); SECTION("Add non-executable blob to storage") { CHECK(cas.StoreBlob(test_bytes, false)); auto file_path = cas.BlobPath(test_digest, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(test_digest, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } SECTION("Add executable blob to storage") { CHECK(cas.StoreBlob(test_bytes, true)); auto file_path = cas.BlobPath(test_digest, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(test_digest, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } } TEST_CASE("LocalCAS: Add blob to storage from non-executable file", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& cas = storage.CAS(); std::filesystem::path non_exec_file{ "test/buildtool/storage/data/non_executable_file"}; auto test_blob = ArtifactDigestFactory::HashFileAs( storage_config.Get().hash_function, non_exec_file); REQUIRE(test_blob); // check blob not in storage CHECK(not cas.BlobPath(*test_blob, true)); CHECK(not cas.BlobPath(*test_blob, false)); // ensure previous calls did not accidentially create the blob CHECK(not cas.BlobPath(*test_blob, true)); CHECK(not cas.BlobPath(*test_blob, false)); SECTION("Add non-executable blob to storage") { CHECK(cas.StoreBlob(non_exec_file, false)); auto file_path = cas.BlobPath(*test_blob, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(*test_blob, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } SECTION("Add executable blob to storage") { CHECK(cas.StoreBlob(non_exec_file, true)); auto file_path = cas.BlobPath(*test_blob, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(*test_blob, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } } TEST_CASE("LocalCAS: Add blob to storage from executable file", "[storage]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); auto const& cas = storage.CAS(); std::filesystem::path exec_file{ "test/buildtool/storage/data/executable_file"}; auto test_blob = ArtifactDigestFactory::HashFileAs( storage_config.Get().hash_function, exec_file); REQUIRE(test_blob); // check blob not in storage CHECK(not cas.BlobPath(*test_blob, true)); CHECK(not cas.BlobPath(*test_blob, false)); // ensure previous calls did not accidentially create the blob CHECK(not cas.BlobPath(*test_blob, true)); CHECK(not cas.BlobPath(*test_blob, false)); SECTION("Add non-executable blob to storage") { CHECK(cas.StoreBlob(exec_file, false)); auto file_path = cas.BlobPath(*test_blob, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(*test_blob, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } SECTION("Add executable blob to storage") { CHECK(cas.StoreBlob(exec_file, true)); auto file_path = cas.BlobPath(*test_blob, false); REQUIRE(file_path); CHECK(FileSystemManager::IsFile(*file_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); auto exe_path = cas.BlobPath(*test_blob, true); REQUIRE(exe_path); CHECK(FileSystemManager::IsFile(*exe_path)); CHECK(FileSystemManager::IsExecutable(*exe_path)); CHECK(not FileSystemManager::IsExecutable(*file_path)); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/system/000077500000000000000000000000001516554100600241175ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/system/TARGETS000066400000000000000000000010111516554100600251440ustar00rootroot00000000000000{ "system_command": { "type": ["@", "rules", "CC/test", "test"] , "name": ["system_command"] , "srcs": ["system_command.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/system", "system_command"] , ["", "catch-main"] ] , "stage": ["test", "buildtool", "system"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["system"] , "deps": ["system_command"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/system/system_command.test.cpp000066400000000000000000000122161516554100600306250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/system/system_command.hpp" #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" namespace { [[nodiscard]] auto GetTestDir() -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/buildtool/file_system"; } } // namespace TEST_CASE("SystemCommand", "[filesystem]") { using Catch::Matchers::Contains; using Catch::Matchers::StartsWith; std::string name{"ExecutorTest"}; SystemCommand system{name}; auto const testdir = GetTestDir(); SECTION("empty command") { auto tmpdir = testdir / "empty"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); auto output = system.Execute( {}, {}, FileSystemManager::GetCurrentDirectory(), tmpdir); CHECK(not output.has_value()); } SECTION("simple command, no arguments, no env variables") { auto tmpdir = testdir / "simple_noargs"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); auto output = system.Execute( {"echo"}, {}, FileSystemManager::GetCurrentDirectory(), tmpdir); REQUIRE(output.has_value()); CHECK(*output == 0); CHECK(*FileSystemManager::ReadFile(tmpdir / "stdout") == "\n"); CHECK(FileSystemManager::ReadFile(tmpdir / "stderr")->empty()); } SECTION( "simple command, env variables are expanded only when wrapped with " "/bin/sh") { auto tmpdir = testdir / "simple_env0"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); auto output = system.Execute({"echo", "${MY_MESSAGE}"}, {{"MY_MESSAGE", "hello"}}, FileSystemManager::GetCurrentDirectory(), tmpdir); REQUIRE(output.has_value()); CHECK(*output == 0); CHECK(*FileSystemManager::ReadFile(tmpdir / "stdout") == "${MY_MESSAGE}\n"); CHECK(FileSystemManager::ReadFile(tmpdir / "stderr")->empty()); tmpdir = testdir / "simple_env1"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); auto output_wrapped = system.Execute({"/bin/sh", "-c", "set -e\necho ${MY_MESSAGE}"}, {{"MY_MESSAGE", "hello"}}, FileSystemManager::GetCurrentDirectory(), tmpdir); REQUIRE(output_wrapped.has_value()); CHECK(*output_wrapped == 0); CHECK(*FileSystemManager::ReadFile(tmpdir / "stdout") == "hello\n"); CHECK(FileSystemManager::ReadFile(tmpdir / "stderr")->empty()); } SECTION("executable, producing std output, std error and return value") { auto tmpdir = testdir / "exe_output"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); auto output = system.Execute( {"/bin/sh", "-c", "set -e\necho this is stdout; echo this is stderr >&2; exit 5"}, {}, FileSystemManager::GetCurrentDirectory(), tmpdir); REQUIRE(output.has_value()); CHECK(*output == 5); CHECK(*FileSystemManager::ReadFile(tmpdir / "stdout") == "this is stdout\n"); CHECK(*FileSystemManager::ReadFile(tmpdir / "stderr") == "this is stderr\n"); } SECTION( "executable dependent on env, producing std output, std error and " "return value") { auto tmpdir = testdir / "exe_output_from_env"; REQUIRE(FileSystemManager::CreateDirectoryExclusive(tmpdir)); std::string const stdout = "this is stdout from env var"; std::string const stderr = "this is stderr from env var"; auto output = system.Execute( {"/bin/sh", "-c", "set -e\necho ${MY_STDOUT}; echo ${MY_STDERR} >&2; exit 5"}, {{"MY_STDOUT", stdout}, {"MY_STDERR", stderr}}, FileSystemManager::GetCurrentDirectory(), tmpdir); REQUIRE(output.has_value()); CHECK(*output == 5); CHECK(*FileSystemManager::ReadFile(tmpdir / "stdout") == stdout + '\n'); CHECK(*FileSystemManager::ReadFile(tmpdir / "stderr") == stderr + '\n'); } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/tree_structure/000077500000000000000000000000001516554100600256525ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/buildtool/tree_structure/TARGETS000066400000000000000000000030241516554100600267050ustar00rootroot00000000000000{ "tree_structure": { "type": ["@", "rules", "CC/test", "test"] , "name": ["tree_structure"] , "srcs": ["tree_structure.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "gsl", "", "gsl"] , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/common", "protocol_traits"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , [ "@" , "src" , "src/buildtool/execution_api/bazel_msg" , "bazel_msg_factory" ] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "git_repo"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "garbage_collector"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/buildtool/tree_structure", "tree_structure_cache"] , ["@", "src", "src/buildtool/tree_structure", "tree_structure_utils"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "hash_combine"] , ["@", "src", "src/utils/cpp", "hex_string"] , ["@", "src", "src/utils/cpp", "path"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "large_object_utils"] , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "tree_structure"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["tree_structure"] , "deps": ["tree_structure"] } } just-buildsystem-justbuild-b1fb5fa/test/buildtool/tree_structure/tree_structure.test.cpp000066400000000000000000000326141516554100600324210ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/tree_structure/tree_structure_cache.hpp" #include "src/buildtool/tree_structure/tree_structure_utils.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/large_objects/large_object_utils.hpp" namespace { [[nodiscard]] auto CreateFlatTestDirectory(StorageConfig const& storage_config, Storage const& storage, std::uintmax_t entries) -> std::optional; [[nodiscard]] auto CreateComplexTestDirectory( StorageConfig const& storage_config, Storage const& storage) -> std::optional; [[nodiscard]] auto ValidateTreeStructure(ArtifactDigest const& digest, Storage const& storage) -> bool; struct TreeEntryHasher { [[nodiscard]] auto operator()( GitRepo::TreeEntry const& entry) const noexcept -> std::size_t { size_t seed{}; hash_combine(&seed, entry.name); hash_combine(&seed, entry.type); return seed; } }; using TreeEntriesHitContainer = std::unordered_map; [[nodiscard]] auto CountTreeEntries( ArtifactDigest const& digest, Storage const& storage, gsl::not_null const& container, bool increment) -> bool; } // namespace TEST_CASE("cache", "[tree_structure]") { auto const storage_config = TestStorageConfig::Create(); if (not ProtocolTraits::IsNative( storage_config.Get().hash_function.GetType())) { return; } auto const storage = Storage::Create(&storage_config.Get()); TreeStructureCache const ts_cache(&storage_config.Get()); auto const from_dir = CreateFlatTestDirectory(storage_config.Get(), storage, 128); REQUIRE(from_dir); auto const to_dir = CreateFlatTestDirectory(storage_config.Get(), storage, 128); REQUIRE(to_dir); // Set dependency REQUIRE(ts_cache.Set(*from_dir, *to_dir)); // Obtain value REQUIRE(ts_cache.Get(*from_dir) == *to_dir); // Reseting dependency fails and the entry doesn't get overwritten: REQUIRE_FALSE(ts_cache.Set(*from_dir, ArtifactDigest{})); REQUIRE(ts_cache.Get(*from_dir) == *to_dir); // Rotate generations REQUIRE(GarbageCollector::TriggerGarbageCollection(storage_config.Get())); auto const youngest = Generation::Create(&storage_config.Get(), 0); // Check there's no entry in the youngest generation: CHECK_FALSE(youngest.CAS().TreePath(*from_dir).has_value()); CHECK_FALSE(youngest.CAS().TreePath(*to_dir).has_value()); // Obtain value one more time and check uplinking has happened: REQUIRE(ts_cache.Get(*from_dir) == *to_dir); CHECK(youngest.CAS().TreePath(*from_dir).has_value()); CHECK(youngest.CAS().TreePath(*to_dir).has_value()); } TEST_CASE("compute", "[tree_structure]") { auto const storage_config = TestStorageConfig::Create(); if (not ProtocolTraits::IsNative( storage_config.Get().hash_function.GetType())) { return; } auto const storage = Storage::Create(&storage_config.Get()); TreeStructureCache const ts_cache(&storage_config.Get()); auto const tree = CreateComplexTestDirectory(storage_config.Get(), storage); REQUIRE(tree); auto const tree_structure = TreeStructureUtils::Compute(*tree, storage, ts_cache); REQUIRE(tree_structure); REQUIRE(ValidateTreeStructure(*tree_structure, storage)); TreeEntriesHitContainer container; // Add recursively all TreeEntries of the source tree to the container, // incrementing counters: REQUIRE(CountTreeEntries(*tree, storage, &container, /*increment=*/true)); // Add recursively all TreeEntries of the tree structure to the container, // decrementing counters: REQUIRE(CountTreeEntries( *tree_structure, storage, &container, /*increment=*/false)); // All counters must be equal to 0, meaning all entries have been hit. auto const all_hit_equally = std::all_of(container.begin(), container.end(), [](auto const& p) { return p.second == 0; }); REQUIRE(all_hit_equally); } namespace { [[nodiscard]] auto CreateDirectory(std::filesystem::path const& directory, Storage const& storage) { BazelMsgFactory::FileStoreFunc store_file = [&storage](std::filesystem::path const& path, bool is_exec) -> std::optional { return storage.CAS().StoreBlob(path, is_exec); }; BazelMsgFactory::TreeStoreFunc store_tree = [&storage]( std::string const& content) -> std::optional { return storage.CAS().StoreTree(content); }; BazelMsgFactory::SymlinkStoreFunc store_symlink = [&storage]( std::string const& content) -> std::optional { return storage.CAS().StoreBlob(content, /*is_executable=*/false); }; return BazelMsgFactory::CreateGitTreeDigestFromLocalTree( directory, store_file, store_tree, store_symlink); } [[nodiscard]] auto CreateFlatTestDirectory(StorageConfig const& storage_config, Storage const& storage, std::uintmax_t entries) -> std::optional { auto tree = storage_config.CreateTypedTmpDir("tree"); if (not tree or not LargeObjectUtils::GenerateDirectory(tree->GetPath(), entries)) { return std::nullopt; } return CreateDirectory(tree->GetPath(), storage); } [[nodiscard]] auto CreateComplexTestDirectory( StorageConfig const& storage_config, Storage const& storage) -> std::optional { auto const test_dir = storage_config.CreateTypedTmpDir("tmp"); auto head_temp_directory = TmpDir::Create(test_dir->GetPath() / "head_dir"); auto const head_temp_dir_path = head_temp_directory->GetPath(); // ├── exec_1 // ├── file_1 // ├── symlink_to_nested_dir_1_1 -> nested_dir_1 / nested_dir_1_1 // ├── symlink_to_nested_dir_2_1 -> nested_dir_2 / nested_dir_2_1 // ├── nested_dir_1 // │ ├── ... // │ ├── nested_dir_1_1 // │ │ └── ... // │ └── nested_dir_1_2 // │ └── ... // └── nested_dir_2 // ├── ... // ├── nested_dir_2_1 // │ └── ... // └── nested_dir_2_2 // └── ... static constexpr std::size_t kFileSize = 128; auto const file_path = head_temp_dir_path / "file_1"; if (not LargeObjectUtils::GenerateFile(file_path, kFileSize)) { return std::nullopt; } auto const exec_path = head_temp_dir_path / "exec_1"; if (not LargeObjectUtils::GenerateFile(exec_path, kFileSize, /*is_executable =*/true)) { return std::nullopt; } std::array const directories = { head_temp_dir_path / "nested_dir_1", head_temp_dir_path / "nested_dir_1" / "nested_dir_1_1", head_temp_dir_path / "nested_dir_1" / "nested_dir_1_2", head_temp_dir_path / "nested_dir_2", head_temp_dir_path / "nested_dir_2" / "nested_dir_2_1", head_temp_dir_path / "nested_dir_2" / "nested_dir_2_2"}; static constexpr std::size_t kDirEntries = 16; for (auto const& path : directories) { if (not LargeObjectUtils::GenerateDirectory(path, kDirEntries)) { return std::nullopt; } } // Create non-upwards symlinks in the top directory: if (not FileSystemManager::CreateNonUpwardsSymlink( std::filesystem::path("nested_dir_1") / "nested_dir_1_1", head_temp_dir_path / "symlink_to_nested_dir_1_1") or not FileSystemManager::CreateNonUpwardsSymlink( std::filesystem::path("nested_dir_2") / "nested_dir_2_1", head_temp_dir_path / "symlink_to_nested_dir_2_1")) { return std::nullopt; } return CreateDirectory(head_temp_dir_path, storage); } [[nodiscard]] auto ReadGitTree(Storage const& storage, ArtifactDigest const& tree) -> std::optional { auto const tree_path = storage.CAS().TreePath(tree); if (not tree_path) { return std::nullopt; } auto const tree_content = FileSystemManager::ReadFile(*tree_path); if (not tree_content) { return std::nullopt; } auto const check_symlinks = [&storage](std::vector const& ids) { return std::all_of( ids.begin(), ids.end(), [&storage](auto const& id) -> bool { auto path_to_symlink = storage.CAS().BlobPath(id, /*is_executable=*/false); if (not path_to_symlink) { return false; } auto const content = FileSystemManager::ReadFile(*path_to_symlink); return content and PathIsNonUpwards(*content); }); }; return GitRepo::ReadTreeData(*tree_content, tree.hash(), check_symlinks, /*is_hex_id=*/true); } [[nodiscard]] auto ValidateTreeStructure(ArtifactDigest const& digest, Storage const& storage) -> bool { auto tree_entries = ReadGitTree(storage, digest); if (not tree_entries) { return false; } auto const empty_blob_hash = ArtifactDigest{}.hash(); for (auto const& [raw_id, es] : *tree_entries) { auto const hex_id = ToHexString(raw_id); for (auto const& entry : es) { switch (entry.type) { case ObjectType::Tree: { auto const git_digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, hex_id, /*size is unknown*/ 0, IsTreeObject(entry.type)); if (not git_digest or not ValidateTreeStructure(*git_digest, storage)) { return false; } } break; default: { if (hex_id != empty_blob_hash) { return false; } } } } } return true; } [[nodiscard]] auto CountTreeEntries( ArtifactDigest const& digest, Storage const& storage, gsl::not_null const& container, bool increment) -> bool { auto tree_entries = ReadGitTree(storage, digest); if (not tree_entries) { return false; } auto const empty_blob_hash = ArtifactDigestFactory::HashDataAs( storage.GetHashFunction(), std::string{}); for (auto const& [raw_id, es] : *tree_entries) { auto const hex_id = ToHexString(raw_id); for (auto const& entry : es) { if (IsTreeObject(entry.type)) { auto const git_digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1, hex_id, /*size is unknown*/ 0, IsTreeObject(entry.type)); if (not git_digest or not CountTreeEntries( *git_digest, storage, container, increment)) { return false; } } increment ? ++(*container)[entry] : --(*container)[entry]; } } return true; } } // namespace just-buildsystem-justbuild-b1fb5fa/test/end-to-end/000077500000000000000000000000001516554100600225305ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/EXPRESSIONS000066400000000000000000000161521516554100600243020ustar00rootroot00000000000000{ "test-action": { "vars": [ "TEST_ENV" , "ATTEMPT" , "name" , "test type name" , "test.sh" , "data" , "extra_infra" , "keep" , "keep-dirs" , "transition" , "TEST_COMPATIBLE_REMOTE" , "TEST_REMOTE_EXECUTION" , "TIMEOUT_SCALE" , "TEST_STANDALONE_SERVE" , "USES_SERVE" ] , "imports": { "artifacts_list": ["@", "rules", "", "field_artifacts_list"] , "runfiles_list": ["@", "rules", "", "field_runfiles_list"] , "artifacts": ["@", "rules", "", "field_artifacts"] , "stage": ["@", "rules", "", "stage_singleton_field"] } , "expression": { "type": "let*" , "bindings": [ ["fieldname", "runner"] , ["location", "runner"] , ["runner", {"type": "CALL_EXPRESSION", "name": "stage"}] , ["fieldname", "just"] , ["just", {"type": "CALL_EXPRESSION", "name": "artifacts"}] , ["fieldname", "deps"] , [ "deps" , { "type": "TREE" , "$1": { "type": "disjoint_map_union" , "msg": "Field \"deps\" has to stage in a conflict free way" , "$1": { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "runfiles_list"} , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} ] } } } ] , [ "attempt marker" , { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "ATTEMPT"} , "$2": null } , "then": {"type": "empty_map"} , "else": { "type": "singleton_map" , "key": "ATTEMPT" , "value": {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}} } } ] , [ "outs" , { "type": "++" , "$1": [ [ "result" , "stdout" , "stderr" , "time-start" , "time-stop" , "pwd" , "remotestdout" , "remotestderr" ] , {"type": "var", "name": "extra_infra"} , { "type": "foreach" , "var": "filename" , "range": {"type": "var", "name": "keep"} , "body": { "type": "join" , "$1": ["work/", {"type": "var", "name": "filename"}] } } ] } ] , [ "out_dirs" , { "type": "++" , "$1": [ { "type": "if" , "cond": {"type": "var", "name": "USES_SERVE"} , "then": { "type": "if" , "cond": {"type": "var", "name": "TEST_STANDALONE_SERVE"} , "then": ["serve"] , "else": ["remote", "serve"] } , "else": ["remote"] } , { "type": "foreach" , "var": "dir_path" , "range": {"type": "var", "name": "keep-dirs"} , "body": { "type": "join" , "$1": ["work/", {"type": "var", "name": "dir_path"}] } } ] } ] , [ "inputs" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "work" , "value": {"type": "var", "name": "deps"} } , { "type": "to_subdir" , "subdir": "staged" , "$1": {"type": "var", "name": "just"} } , {"type": "var", "name": "runner"} , {"type": "var", "name": "test.sh"} , {"type": "var", "name": "data"} , {"type": "var", "name": "attempt marker"} ] } ] , [ "cmd" , { "type": "++" , "$1": [ [ "./runner" , { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": "true" , "else": "false" } , { "type": "json_encode" , "$1": {"type": "var", "name": "TEST_REMOTE_EXECUTION"} } ] , {"type": "var", "name": "keep"} ] } ] , [ "test_env" , {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} ] ] , "body": { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null} , "then": { "type": "ACTION" , "outs": {"type": "var", "name": "outs"} , "out_dirs": {"type": "var", "name": "out_dirs"} , "inputs": {"type": "var", "name": "inputs"} , "cmd": {"type": "var", "name": "cmd"} , "env": {"type": "var", "name": "test_env"} , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "may_fail": ["test"] , "fail_message": { "type": "join" , "$1": [ "shell test with " , { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": "compatible" , "else": "native" } , " " , {"type": "var", "name": "test type name"} , " " , {"type": "var", "name": "name"} , " failed" ] } } , "else": { "type": "ACTION" , "outs": {"type": "var", "name": "outs"} , "out_dirs": {"type": "var", "name": "out_dirs"} , "inputs": {"type": "var", "name": "inputs"} , "cmd": {"type": "var", "name": "cmd"} , "env": {"type": "var", "name": "test_env"} , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "may_fail": ["test"] , "no_cache": ["test"] , "fail_message": { "type": "join" , "$1": [ "shell test with " , {"type": "var", "name": "test type name"} , " " , {"type": "var", "name": "name"} , " failed (Run " , {"type": "var", "name": "ATTEMPT"} , ")" ] } } } } } , "test-result": { "vars": [ "TEST_ENV" , "name" , "test type name" , "test.sh" , "data" , "extra_infra" , "keep" , "keep-dirs" , "transition" , "TEST_COMPATIBLE_REMOTE" , "TEST_REMOTE_EXECUTION" , "TIMEOUT_SCALE" , "TEST_STANDALONE_SERVE" , "USES_SERVE" ] , "imports": {"action": "test-action"} , "expression": { "type": "let*" , "bindings": [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "test-results"} , "runfiles": {"type": "var", "name": "runfiles"} } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/RULES000066400000000000000000000561231516554100600233540ustar00rootroot00000000000000{ "with remote": { "doc": [ "Shell test, given by a test script," , "implicitly assuming a remote execution in the background." ] , "target_fields": ["deps", "test"] , "string_fields": ["keep", "name", "keep-dirs"] , "config_vars": [ "ARCH" , "HOST_ARCH" , "RUNS_PER_TEST" , "TEST_ENV" , "TEST_COMPATIBLE_REMOTE" , "TEST_REMOTE_EXECUTION" , "TIMEOUT_SCALE" , "TEST_SUMMARY_EXECUTION_PROPERTIES" ] , "field_doc": { "test": [ "The shell script for the test, launched with sh." , "" , "The script will be invoked in the specified TEST_ENV with the" , "following variables added." , " - REMOTE_EXECUTION_ADDRESS containing the IP:port pair the" , " implicit remote-execution endpoint is listening at." , " - COMPATIBLE is set to an non-empty string, if the server was" , " started in compatible mode, and unset otherwise." , " - REMOTE_BIN specifying an extra directory added to remote-execution" , " actions early in path." , "Additionally, the variables TLS_CA_CERT, TLS_CLIENT_CERT, and" , "TLS_CLIENT_KEY are removed from the environment." , "" , "Moreover, an empty directory is created for any temporary files needed" , "by the test, and it is made available in the environment variable" , "TEST_TMPDIR. The test should not assume write permissions" , "outside the working directory and the TEST_TMPDIR." , "For convenience, the environment variable TMPDIR is also set to" , "TEST_TMPDIR." ] , "name": [ "A name for the test, used in reporting, as well as for staging" , "the test result tree in the runfiles" ] , "keep": [ "List of names (relative to the test working directory) of files that" , "the test might generate that should be kept as part of the output." , "This might be useful for further analysis of the test" ] , "keep-dirs": [ "List of names (relative to the test working directory) of directories" , "that the test might generate that should be kept as part of the" , "output. This might be useful for further analysis of the test" ] , "deps": [ "Any targets that should be staged (with artifacts and runfiles) into" , "the tests working directory" ] } , "config_doc": { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TEST_COMPATIBLE_REMOTE": ["If true, run the remote execution in compatible mode."] , "TEST_REMOTE_EXECUTION": [ "JSON object (aka map) containing at least \"interface\" and \"port\"." , "If the remote execution service requires additional arguments, they can be listed" , "in the list of strings named \"args\"." ] , "TIMEOUT_SCALE": ["Factor on how to scale the timeout for this test. Defaults to 1.0."] , "TEST_SUMMARY_EXECUTION_PROPERTIES": [ "Additional remote-execution properties for the test-summarizing action" , "in case RUNS_PER_TEST is set; defaults to the empty map." ] } , "tainted": ["test"] , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "remotestdout/remotestderr: Any output of the remote-execution server" , " implicit to that test" , "work: In this directory, all the files specified to \"keep\" and" , " \"keep-dirs\" are staged" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] , "implicit": { "runner": ["with_remote_test_runner.py"] , "summarizer": [["@", "rules", "shell/test", "summarizer"]] , "just": [["", "tool-under-test"]] } , "imports": { "test-result": "test-result" , "action": "test-action" , "stage": ["@", "rules", "", "stage_singleton_field"] , "host transition": ["@", "rules", "transitions", "for host"] , "field_list": ["@", "rules", "", "field_list_provider"] } , "config_transitions": { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "just": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "summarizer": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ ["test type name", "remote execution"] , ["USES_SERVE", false] , ["extra_infra", []] , ["data", {"type": "empty_map"}] , [ "test.sh" , { "type": "context" , "msg": "Expecting 'test' to specify precisely one file containing a shell script" , "$1": { "type": "let*" , "bindings": [["fieldname", "test"], ["location", "test.sh"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } } ] , [ "name" , { "type": "assert_non_empty" , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["keep", {"type": "FIELD", "name": "keep"}] , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}] , ["deps-fieldname", "deps"] , ["transition", {"type": "CALL_EXPRESSION", "name": "host transition"}] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "RUNS_PER_TEST"} , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} , "then": { "type": "let*" , "bindings": [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "ATTEMPT" , "range": { "type": "range" , "$1": {"type": "var", "name": "RUNS_PER_TEST"} } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} , "value": {"type": "CALL_EXPRESSION", "name": "action"} } } } ] , [ "summarizer" , { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "summary artifacts" , { "type": "++" , "$1": [ ["result"] , { "type": "let*" , "bindings": [["provider", "artifacts"], ["fieldname", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} } ] } ] , [ "attempts (for summary)" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "summary artifacts"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "$_"} , "key": {"type": "var", "name": "_"} } } } } } } } } ] , [ "summary" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } , "outs": ["stdout", "stderr", "result", "time-start", "time-stop"] , "cmd": ["./summarizer"] , "execution properties": { "type": "var" , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" , "default": {"type": "empty_map"} } } ] , [ "attempts" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "$_"}} } } } ] , [ "artifacts" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "pwd" , "value": {"type": "BLOB", "data": "/summary"} } , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} } ] } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "artifacts"} , "runfiles": {"type": "var", "name": "runfiles"} } } } } } , "with serve": { "doc": [ "Shell test, given by a test script," , "implicitly assuming a remote execution and a just serve instance" , "in the background." ] , "target_fields": ["deps", "test", "repos"] , "string_fields": ["keep", "keep-dirs", "name"] , "config_vars": [ "ARCH" , "HOST_ARCH" , "RUNS_PER_TEST" , "TEST_ENV" , "TEST_COMPATIBLE_REMOTE" , "TEST_STANDALONE_SERVE" , "TIMEOUT_SCALE" , "TEST_SUMMARY_EXECUTION_PROPERTIES" ] , "field_doc": { "test": [ "The shell script for the test, launched with sh." , "" , "The script will be invoked in the specified TEST_ENV with the" , "following variables added." , " - REMOTE_EXECUTION_ADDRESS containing the IP:port pair the" , " implicit remote-execution endpoint is listening at." , " - COMPATIBLE is set to an non-empty string, if the server was" , " started in compatible mode, and unset otherwise." , " - REMOTE_LBR containing the absolute path to the local build root" , " of the implicit remote-execution end point" , " - SERVE_ADDRESS containing the IP:port pair the" , " implicit serve endpoint is listening at." , " - SERVE_LBR containing the absolute path to the local build root" , " of the implicit serve end point" , " - COMMIT_0, COMMIT_1, ... containing the commit id of the head" , " commit of the repositories available to the serve endpoint" , " - TREE_0, TREE_1, ... containing the git tree identifiers of" , " the head commit of the repositories available ot the serve" , " endpoint" , "Additionally, the variables TLS_CA_CERT, TLS_CLIENT_CERT, and" , "TLS_CLIENT_KEY are removed from the environment." , "" , "Moreover, an empty directory is created for any temporary files needed" , "by the test, and it is made available in the environment variable" , "TEST_TMPDIR. The test should not assume write permissions" , "write permissions outside the working directory, the TEST_TMPDIR," , "REMOTE_LBR, and SERVE_LBR." , "For convenience, the environment variable TMPDIR is also set to" , "TEST_TMPDIR." ] , "name": [ "A name for the test, used in reporting, as well as for staging" , "the test result tree in the runfiles" ] , "keep": [ "List of names (relative to the test working directory) of files that" , "the test might generate that should be kept as part of the output." , "This might be useful for further analysis of the test" ] , "keep-dirs": [ "List of names (relative to the test working directory) of directories" , "that the test might generate that should be kept as part of the" , "output. This might be useful for further analysis of the test" ] , "deps": [ "Any targets that should be staged (with artifacts and runfiles) into" , "the tests working directory" ] , "repos": [ "The trees, one per entry, that the just serve instance should have" , "available. The respective commits will be generated and passed to the" , "test script as COMMIT_0, COMMIT_1, etc., while their respective root" , "tree identifiers are passed to the test script as TREE_0, TREE_1, etc." ] } , "config_doc": { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TEST_COMPATIBLE_REMOTE": ["If true, run the remote execution in compatible mode."] , "TIMEOUT_SCALE": ["Factor on how to scale the timeout for this test. Defaults to 1.0."] , "TEST_SUMMARY_EXECUTION_PROPERTIES": [ "Additional remote-execution properties for the test-summarizing action" , "in case RUNS_PER_TEST is set; defaults to the empty map." ] } , "tainted": ["test"] , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "remotestdout/remotestderr: Any output of the remote-execution server" , " implicit to that test" , "servestdout/servestderr: Any output of the serve-execution server" , " implicit to that test" , "work: In this directory, all the files specified to \"keep\" are staged" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] , "implicit": { "runner": ["with_serve_test_runner.py"] , "summarizer": [["@", "rules", "shell/test", "summarizer"]] , "just": [["", "tool-under-test"]] } , "imports": { "test-result": "test-result" , "action": "test-action" , "stage": ["@", "rules", "", "stage_singleton_field"] , "host transition": ["@", "rules", "transitions", "for host"] , "field_list": ["@", "rules", "", "field_list_provider"] } , "config_transitions": { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "repos": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "just": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "summarizer": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ ["test type name", "target-level cache serving"] , ["USES_SERVE", true] , ["extra_infra", ["servestdout", "servestderr"]] , [ "test.sh" , { "type": "context" , "msg": "Expecting 'test' to specify precisely one file containing a shell script" , "$1": { "type": "let*" , "bindings": [["fieldname", "test"], ["location", "test.sh"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } } ] , [ "name" , { "type": "assert_non_empty" , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["keep", {"type": "FIELD", "name": "keep"}] , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}] , ["deps-fieldname", "deps"] , ["transition", {"type": "CALL_EXPRESSION", "name": "host transition"}] , [ "data" , { "type": "to_subdir" , "subdir": "data" , "$1": { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "enumerate", "$1": {"type": "FIELD", "name": "repos"}} , "body": { "type": "to_subdir" , "subdir": {"type": "var", "name": "_"} , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "$_"} , "transition": {"type": "var", "name": "transition"} } } } } } ] , [ "TEST_REMOTE_EXECUTION" , { "type": "if" , "cond": {"type": "var", "name": "TEST_STANDALONE_SERVE"} , "then": true , "else": false } ] ] , "body": { "type": "if" , "cond": {"type": "var", "name": "RUNS_PER_TEST"} , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} , "then": { "type": "let*" , "bindings": [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "ATTEMPT" , "range": { "type": "range" , "$1": {"type": "var", "name": "RUNS_PER_TEST"} } , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} , "value": {"type": "CALL_EXPRESSION", "name": "action"} } } } ] , [ "summarizer" , { "type": "let*" , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] , [ "summary artifacts" , { "type": "++" , "$1": [ ["result"] , { "type": "let*" , "bindings": [["provider", "artifacts"], ["fieldname", "summarizer"]] , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} } ] } ] , [ "attempts (for summary)" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "var", "name": "summary artifacts"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": { "type": "lookup" , "map": {"type": "var", "name": "$_"} , "key": {"type": "var", "name": "_"} } } } } } } } } ] , [ "summary" , { "type": "ACTION" , "inputs": { "type": "map_union" , "$1": [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } , "outs": ["stdout", "stderr", "result", "time-start", "time-stop"] , "cmd": ["./summarizer"] , "execution properties": { "type": "var" , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" , "default": {"type": "empty_map"} } } ] , [ "attempts" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "attempts (plain)"} , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "_"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "$_"}} } } } ] , [ "artifacts" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "pwd" , "value": {"type": "BLOB", "data": "/summary"} } , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} } ] } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "artifacts"} , "runfiles": {"type": "var", "name": "runfiles"} } } } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/TARGETS000066400000000000000000000054651516554100600235760ustar00rootroot00000000000000{ "git-import-under-test": { "type": "install" , "files": {"bin/git-import-under-test": ["@", "src", "", "bin/just-import-git.py"]} } , "deduplicate-tool-under-test": { "type": "install" , "files": { "bin/deduplicate-tool-under-test": ["@", "src", "", "bin/just-deduplicate-repos.py"] } } , "lock-tool-under-test": { "type": "install" , "files": {"bin/lock-tool-under-test": ["@", "src", "", "bin/just-lock.py"]} } , "remote tests (unconfigured)": { "type": ["@", "rules", "test", "matrix"] , "arguments_config": ["TEST_COMPATIBLE_REMOTE", "TEST_BOOTSTRAP_JUST_MR"] , "deps": { "type": "++" , "$1": [ [["./", "remote-execution", "TESTS"], ["./", "just-mr", "TESTS"]] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "else": [ ["./", "serve-service", "TESTS"] , ["./", "computed-roots", "TESTS"] , ["./", "tree-structure", "TESTS"] , ["./", "symlinks", "TESTS"] ] } ] } } , "remote tests": { "type": "configure" , "tainted": ["test"] , "target": "remote tests (unconfigured)" , "config": { "type": "'" , "$1": { "TEST_MATRIX": { "TEST_COMAPTIBLE_REMOTE": {"remote-compat": true, "remote-native": "false"} } } } } , "just-mr tests (unconfigured)": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "deps": { "type": "`" , "$1": [ "remote tests" , ["./", "built-in-rules", "TESTS"] , ["./", "cli", "TESTS"] , ["./", "just-lock", "TESTS"] , ["./", "target-cache", "TESTS"] , ["./", "gc", "TESTS"] , ["./", "git-import", "TESTS"] , { "type": ",@" , "$1": { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "else": [["./", "profile", "TESTS"]] } } ] } } , "just-mr tests (matrix)": { "type": ["@", "rules", "test", "matrix"] , "deps": ["just-mr tests (unconfigured)"] } , "just-mr tests": { "type": "configure" , "tainted": ["test"] , "target": "just-mr tests (matrix)" , "config": { "type": "'" , "$1": { "TEST_MATRIX": { "TEST_BOOTSTRAP_JUST_MR": {"just-mr-bootstrap": true, "just-mr-native": false} } } } } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "stage": ["end-to-end"] , "deps": { "type": "`" , "$1": [ "just-mr tests" , ["./", "actions", "TESTS"] , ["./", "build-fails", "TESTS"] , ["./", "execution-service", "TESTS"] , ["./", "generated-binary", "TESTS"] , ["./", "target-tests", "TESTS"] , ["./", "user-errors", "TESTS"] ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/000077500000000000000000000000001516554100600241705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/TARGETS000066400000000000000000000071261516554100600252320ustar00rootroot00000000000000{ "equality": { "type": ["@", "rules", "shell/test", "script"] , "name": ["equality"] , "test": ["action-equality.sh"] , "keep": ["graph.json"] , "deps": [["", "tool-under-test"]] } , "equality-timeout": { "type": ["@", "rules", "shell/test", "script"] , "name": ["equality-timeout"] , "test": ["action-equality-timeout.sh"] , "keep": [ "rules/graph-null.json" , "rules/graph-1.json" , "rules/graph-2.json" , "generic/graph-null.json" , "generic/graph-1.json" , "generic/graph-2.json" ] , "deps": [["", "tool-under-test"]] } , "equality-properties": { "type": ["@", "rules", "shell/test", "script"] , "name": ["equality-properties"] , "test": ["action-equality-properties.sh"] , "keep": [ "rules/graph-null.json" , "rules/graph-empty.json" , "rules/graph-set.json" , "generic/graph-null.json" , "generic/graph-empty.json" , "generic/graph-set.json" ] , "deps": [["", "tool-under-test"]] } , "trees": { "type": ["@", "rules", "shell/test", "script"] , "name": ["trees"] , "test": ["nested-trees.sh"] , "keep": ["blobs.json", "trees.json", "out/index.txt"] , "deps": [["", "tool-under-test"]] } , "conflicts": { "type": ["@", "rules", "shell/test", "script"] , "name": ["conflicts"] , "test": ["conflicts.sh"] , "deps": [["", "tool-under-test"]] } , "incomplete-retry": { "type": ["end-to-end", "with remote"] , "name": ["incompete-retry"] , "test": ["incomplete-retry.sh"] , "deps": [["", "tool-under-test"]] } , "error-reporting": { "type": ["@", "rules", "shell/test", "script"] , "name": ["error-reporting"] , "test": ["error-reporting.sh"] , "deps": [["", "tool-under-test"]] } , "identical-inputs (real)": { "type": ["@", "rules", "shell/test", "script"] , "name": ["identical-inputs"] , "test": ["identical-inputs.sh"] , "deps": [["", "tool-under-test"]] } , "identical-inputs": { "type": "configure" , "tainted": ["test"] , "target": "identical-inputs (real)" , "arguments_config": ["TIMEOUT_SCALE"] , "config": { "type": "singleton_map" , "key": "TIMEOUT_SCALE" , "value": { "type": "*" , "$1": [3, {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0}] } } } , "cwd": { "type": ["@", "rules", "shell/test", "script"] , "name": ["cwd"] , "test": ["cwd.sh"] , "deps": [["", "tool-under-test"]] } , "request-action-input": { "type": ["@", "rules", "shell/test", "script"] , "name": ["request-action-input"] , "test": ["request-action-input.sh"] , "deps": [["", "tool-under-test"]] } , "tree-ops": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tree-ops"] , "test": ["tree-ops.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/graph.json", "out/artifacts.json"] } , "tree-conflicts": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tree-conflicts"] , "test": ["tree-conflicts.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/graph.json", "out/artifacts.json", "out/log"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["DROP_IO_HEAVY_TESTS"] , "stage": ["actions"] , "deps": { "type": "++" , "$1": [ [ "equality" , "equality-timeout" , "equality-properties" , "trees" , "tree-ops" , "tree-conflicts" , "conflicts" , "incomplete-retry" , "error-reporting" , "cwd" , "request-action-input" ] , { "type": "if" , "cond": {"type": "var", "name": "DROP_IO_HEAVY_TESTS"} , "else": ["identical-inputs"] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/action-equality-properties.sh000077500000000000000000000067411516554100600320410ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" mkdir -p "${ROOT}/rules" cd "${ROOT}/rules" touch ROOT cat > RULES <<'EOI' { "": { "config_vars": ["PROPERTIES"] , "expression": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "outs": ["out.txt"] , "cmd": ["sh", "-c", "echo Hello > out.txt\n"] , "execution properties": {"type": "var", "name": "PROPERTIES"} } } } } EOI cat > TARGETS <<'EOI' { "flexible": {"type": ""} , "fixed": { "type": "configure" , "target": "flexible" , "config": { "type": "singleton_map" , "key": "PROPERTIES" , "value": {"type": "empty_map"} } } , "unset": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "PROPERTIES", "value": null} } , "": { "type": "install" , "files": {"flexible": "flexible", "fixed": "fixed", "unset": "unset"} } } EOI mkdir -p "${ROOT}/generic" cd "${ROOT}/generic" touch ROOT cat > TARGETS <<'EOI' { "flexible": { "type": "generic" , "arguments_config": ["PROPERTIES"] , "outs": ["out.txt"] , "cmds": ["echo Hello > out.txt"] , "execution properties": {"type": "var", "name": "PROPERTIES"} } , "fixed": { "type": "configure" , "target": "flexible" , "config": { "type": "singleton_map" , "key": "PROPERTIES" , "value": {"type": "empty_map"} } } , "unset": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "PROPERTIES", "value": null} } , "": { "type": "install" , "files": {"flexible": "flexible", "fixed": "fixed", "unset": "unset"} } } EOI for variant in rules generic do echo "Testing variant ${variant}" echo cd "${ROOT}/${variant}" "${JUST}" analyse --local-build-root "${LBRDIR}" --dump-graph graph-null.json 2>&1 action_configs=$(cat graph-null.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .config.PROPERTIES ] | sort] | sort') echo "${action_configs}" [ "${action_configs}" = '[[null,{}]]' ] echo "${JUST}" analyse --local-build-root "${LBRDIR}" -D '{"PROPERTIES": {}}' --dump-graph graph-empty.json 2>&1 action_configs=$(cat graph-empty.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .config.PROPERTIES ] | sort ] | sort') echo "${action_configs}" [ "${action_configs}" = '[[null,{}]]' ] "${JUST}" analyse --local-build-root "${LBRDIR}" -D '{"PROPERTIES": {"foo": "bar"}}' --dump-graph graph-set.json 2>&1 action_configs=$(cat graph-set.json | jq -acM '.actions | [ .[] | .origins | sort | [ .[] | .config.PROPERTIES ] | sort] | sort') echo "${action_configs}" [ "${action_configs}" = '[[null,{}],[{"foo":"bar"}]]' ] echo "variant ${variant} OK" echo done # also verify the consistency of the generated actions for graph in graph-null.json graph-empty.json graph-set.json do diff -u "${ROOT}/rules/${graph}" "${ROOT}/generic/${graph}" done echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/action-equality-timeout.sh000077500000000000000000000065271516554100600313350ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" mkdir -p "${ROOT}/rules" cd "${ROOT}/rules" touch ROOT cat > RULES <<'EOI' { "": { "config_vars": ["TIMEOUT"] , "expression": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "outs": ["out.txt"] , "cmd": ["sh", "-c", "echo Hello > out.txt\n"] , "timeout scaling": {"type": "var", "name": "TIMEOUT"} } } } } EOI cat > TARGETS <<'EOI' { "flexible": {"type": ""} , "fixed": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "TIMEOUT", "value": 1.0} } , "unset": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "TIMEOUT", "value": null} } , "": { "type": "install" , "files": {"flexible": "flexible", "fixed": "fixed", "unset": "unset"} } } EOI mkdir -p "${ROOT}/generic" cd "${ROOT}/generic" touch ROOT cat > TARGETS <<'EOI' { "flexible": { "type": "generic" , "arguments_config": ["TIMEOUT"] , "outs": ["out.txt"] , "cmds": ["echo Hello > out.txt"] , "timeout scaling": {"type": "var", "name": "TIMEOUT"} } , "fixed": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "TIMEOUT", "value": 1.0} } , "unset": { "type": "configure" , "target": "flexible" , "config": {"type": "singleton_map", "key": "TIMEOUT", "value": null} } , "": { "type": "install" , "files": {"flexible": "flexible", "fixed": "fixed", "unset": "unset"} } } EOI for variant in rules generic do echo "Testing variant ${variant}" echo cd "${ROOT}/${variant}" "${JUST}" analyse --local-build-root "${LBRDIR}" --dump-graph graph-null.json 2>&1 cat graph-null.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .config.TIMEOUT ]] | sort' > action_configs cat action_configs [ "$(jq -acM '. == [[1,null]]' action_configs)" = "true" ] echo "${JUST}" analyse --local-build-root "${LBRDIR}" -D '{"TIMEOUT": 1.0}' --dump-graph graph-1.json 2>&1 cat graph-1.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .config.TIMEOUT ]] | sort' > action_configs cat action_configs [ "$(jq -acM '. == [[1,null]]' action_configs)" = "true" ] echo "${JUST}" analyse --local-build-root "${LBRDIR}" -D '{"TIMEOUT": 2.0}' --dump-graph graph-2.json 2>&1 cat graph-2.json cat graph-2.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .config.TIMEOUT ]] | sort' > action_configs cat action_configs [ "$(jq -acM '. == [[1,null],[2]]' action_configs)" = "true" ] echo "variant ${variant} OK" echo done # also verify the consistency of the generated actions for graph in graph-null.json graph-1.json graph-2.json do diff -u "${ROOT}/rules/${graph}" "${ROOT}/generic/${graph}" done echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/action-equality.sh000077500000000000000000000041741516554100600276450ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir .tool-root touch ROOT cat > TARGETS <<'EOI' { "foo": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo Hello World > out.txt"] } , "bar": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo Hello World > out.txt"] } , "baz": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo -n Hello > out.txt && echo ' World' >> out.txt"] } , "foo upper": { "type": "generic" , "deps": ["foo"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "bar upper": { "type": "generic" , "deps": ["bar"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "baz upper": { "type": "generic" , "deps": ["baz"] , "outs": ["upper.txt"] , "cmds": ["cat out.txt | tr a-z A-Z > upper.txt"] } , "ALL": { "type": "install" , "files": {"foo.txt": "foo upper", "bar.txt": "bar upper", "baz.txt": "baz upper"} } } EOI bin/tool-under-test build -L '["env", "PATH='"${PATH}"'"]' \ -J 1 --local-build-root .tool-root -f build.log --log-limit 2 2>&1 cat build.log echo grep 'Processed.* 4 actions' build.log grep '1 cache hit' build.log echo bin/tool-under-test analyse --local-build-root .tool-root --dump-graph graph.json 2>&1 echo matching_targets=$(cat graph.json | jq -acM '.actions | [ .[] | .origins | [ .[] | .target]] | sort') echo "${matching_targets}" [ "${matching_targets}" = '[[["@","","","bar"],["@","","","foo"]],[["@","","","bar upper"],["@","","","foo upper"]],[["@","","","baz"]],[["@","","","baz upper"]]]' ] just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/conflicts.sh000077500000000000000000000175341516554100600265250ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly LBRDIR="$TMPDIR/local-build-root" touch ROOT cat > RULES <<'EOI' { "no-conflict": { "expression": { "type": "let*" , "bindings": [ [ "map" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo.txt" , "value": {"type": "BLOB", "data": "Same String"} } , { "type": "singleton_map" , "key": "./foo.txt" , "value": {"type": "BLOB", "data": "Same String"} } , { "type": "singleton_map" , "key": "bar/baz/../../not-upwards.txt" , "value": {"type": "BLOB", "data": "unrelated"} } ] } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "map"} , "outs": ["out"] , "cmd": ["cp", "foo.txt", "out"] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } , "input-conflict": { "expression": { "type": "let*" , "bindings": [ [ "map" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo.txt" , "value": {"type": "BLOB", "data": "Some content"} } , { "type": "singleton_map" , "key": "./foo.txt" , "value": {"type": "BLOB", "data": "A different content"} } ] } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "map"} , "outs": ["out"] , "cmd": ["cp", "foo.txt", "out"] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } , "artifacts-conflict": { "expression": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo.txt" , "value": {"type": "BLOB", "data": "Some content"} } , { "type": "singleton_map" , "key": "./foo.txt" , "value": {"type": "BLOB", "data": "A different content"} } ] } } } , "runfiles-conflict": { "expression": { "type": "RESULT" , "runfiles": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo.txt" , "value": {"type": "BLOB", "data": "Some content"} } , { "type": "singleton_map" , "key": "./foo.txt" , "value": {"type": "BLOB", "data": "A different content"} } ] } } } , "input-tree-conflict": { "expression": { "type": "let*" , "bindings": [ [ "map" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo" , "value": {"type": "TREE"} } , { "type": "singleton_map" , "key": "./foo/bar.txt" , "value": {"type": "BLOB"} } ] } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "map"} , "outs": ["out"] , "cmd": ["cp", "foo.txt", "out"] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } , "artifacts-tree-conflict": { "expression": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": [ {"type": "singleton_map", "key": "foo", "value": {"type": "TREE"}} , { "type": "singleton_map" , "key": "./foo/bar.txt" , "value": {"type": "BLOB"} } ] } } } , "runfiles-tree-conflict": { "expression": { "type": "RESULT" , "runfiles": { "type": "map_union" , "$1": [ {"type": "singleton_map", "key": "foo", "value": {"type": "TREE"}} , { "type": "singleton_map" , "key": "./foo/bar.txt" , "value": {"type": "BLOB"} } ] } } } , "file-file-conflict": { "expression": { "type": "let*" , "bindings": [ [ "map" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "foo/bar/baz.txt" , "value": {"type": "BLOB", "data": "Some content"} } , { "type": "singleton_map" , "key": "foo/bar/baz.txt/other/file.txt" , "value": {"type": "BLOB", "data": "A different content"} } ] } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "map"} , "outs": ["out"] , "cmd": ["cp", "foo.txt", "out"] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } , "upwards-inputs": { "expression": { "type": "let*" , "bindings": [ [ "map" , { "type": "singleton_map" , "key": "bar/../../foo.txt" , "value": {"type": "BLOB", "data": "Some content"} } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "map"} , "outs": ["out"] , "cmd": ["cp", "../foo.txt", "out"] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } } EOI cat > TARGETS <<'EOI' { "no-conflict": {"type": "no-conflict"} , "input-conflict": {"type": "input-conflict"} , "artifacts-conflict": {"type": "artifacts-conflict"} , "runfiles-conflict": {"type": "runfiles-conflict"} , "input-tree-conflict": {"type": "input-tree-conflict"} , "artifacts-tree-conflict": {"type": "artifacts-tree-conflict"} , "runfiles-tree-conflict": {"type": "runfiles-tree-conflict"} , "file-file-conflict": {"type": "file-file-conflict"} , "upwards-inputs": {"type": "upwards-inputs"} } EOI echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" no-conflict --dump-actions - 2>&1 echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log input-conflict 2>&1 && exit 1 || : grep -i 'error.*input.*conflict.*foo\.txt' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log artifacts-conflict 2>&1 && exit 1 || : grep -i 'error.*artifacts.*conflict.*foo\.txt' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log runfiles-conflict 2>&1 && exit 1 || : grep -i 'error.*runfiles.*conflict.*foo\.txt' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log input-tree-conflict 2>&1 && exit 1 || : grep -i 'error.*input.*conflict.*tree.*foo' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log artifacts-tree-conflict 2>&1 && exit 1 || : grep -i 'error.*artifacts.*conflict.*tree.*foo' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log runfiles-tree-conflict 2>&1 && exit 1 || : grep -i 'error.*runfiles.*conflict.*tree.*foo' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log file-file-conflict 2>&1 && exit 1 || : grep -i 'error.*.*conflict.*foo/bar/baz.txt' log echo ./bin/tool-under-test analyse --local-build-root "$LBRDIR" -f log upwards-inputs 2>&1 && exit 1 || : grep -i 'error.*\.\./foo.txt' log echo echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/cwd.sh000066400000000000000000000100651516554100600253030ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${TMPDIR}/out" mkdir -p "${OUT}" readonly JUST="${ROOT}/bin/tool-under-test" mkdir work cd work touch ROOT cat > RULES <<'EOF' { "action": { "string_fields": ["cmd", "cwd", "outs"] , "target_fields": ["deps"] , "expression": { "type": "let*" , "bindings": [ [ "deps" , { "type": "map_union" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} } } ] , [ "cwd" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "cwd"} } ] , [ "action" , { "type": "ACTION" , "inputs": {"type": "var", "name": "deps"} , "outs": {"type": "FIELD", "name": "outs"} , "cmd": {"type": "FIELD", "name": "cmd"} , "cwd": {"type": "var", "name": "cwd"} } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "action"}} } } } EOF cat > TARGETS <<'EOF' { "inputs": { "type": "install" , "files": {"tools/bin/generate.sh": "generate.sh", "work/name.txt": "name.txt"} } , "rule": { "type": "action" , "cmd": ["sh", "../tools/bin/generate.sh", "name.txt"] , "cwd": ["work"] , "outs": ["work/hello.txt", "log/log.txt"] , "deps": ["inputs"] } , "generic": { "type": "generic" , "cmds": ["sh ../tools/bin/generate.sh name.txt"] , "cwd": "work" , "outs": ["work/hello.txt", "log/log.txt"] , "deps": ["inputs"] } , "rule-unrelated-cwd": { "type": "action" , "cmd": ["sh", "../tools/bin/generate.sh", "../work/name.txt", "../work/hello.txt"] , "cwd": ["unrelated"] , "outs": ["work/hello.txt", "log/log.txt"] , "deps": ["inputs"] } , "generic-unrelated-cwd": { "type": "generic" , "cmds": ["sh ../tools/bin/generate.sh ../work/name.txt ../work/hello.txt"] , "cwd": "unrelated" , "outs": ["work/hello.txt", "log/log.txt"] , "deps": ["inputs"] } } EOF cat > generate.sh <<'EOF' echo -n 'Hello ' > ${2:-hello.txt} cat $1 >> ${2:-hello.txt} echo '!' >> ${2:-hello.txt} mkdir -p ../log echo DoNe >> ../log/log.txt EOF echo -n WoRlD > name.txt for target in rule generic rule-unrelated-cwd generic-unrelated-cwd do echo echo Action obtained for target $target echo "${JUST}" analyse \ --local-build-root "${LBR}" \ --dump-actions "${OUT}/${target}-actions.json" \ "$target" 2>&1 echo cat "${OUT}/${target}-actions.json" echo echo echo Verify correct action execution for target $target echo "${JUST}" install \ --local-build-root "${LBR}" \ --local-launcher '["env", "PATH='"${PATH}"'"]' \ -o "${OUT}/$target" "$target" 2>&1 grep WoRlD "${OUT}/$target/work/hello.txt" grep DoNe "${OUT}/$target/log/log.txt" done # Verify the input trees [ $(jq '.[0].input | keys | . == ["tools/bin/generate.sh", "work/name.txt"]' "${OUT}/rule-actions.json") = true ] [ $(jq '.[0].input | keys | . == ["tools/bin/generate.sh", "work/name.txt"]' "${OUT}/generic-actions.json") = true ] [ $(jq '.[0].input | keys | . == ["tools/bin/generate.sh", "unrelated", "work/name.txt"]' "${OUT}/rule-unrelated-cwd-actions.json") = true ] [ $(jq '.[0].input | keys | . == ["tools/bin/generate.sh", "unrelated", "work/name.txt"]' "${OUT}/generic-unrelated-cwd-actions.json") = true ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/error-reporting.sh000066400000000000000000000046541516554100600276750ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${TMPDIR}/out" readonly JUST="${ROOT}/bin/tool-under-test" readonly LOCAL_TOOLS="${TMPDIR}/tools" touch ROOT cat > TARGETS <<'EOF' { "theTargetCausingTheFailure": { "type": "generic" , "arguments_config": ["foo", "bar", "magic"] , "outs": ["a.txt"] , "cmds": [ { "type": "join" , "$1": [ "echo -n stdout-of-; echo failing-target-" , {"type": "var", "name": "foo", "default": ""} , {"type": "var", "name": "bar", "default": ""} , "-$MAGIC_VAR" ] } , "touch a.txt" , "exit 42" ] , "env": { "type": "singleton_map" , "key": "MAGIC_VAR" , "value": {"type": "var", "name": "magic", "default": "unknown"} } } } EOF cat TARGETS echo # Verify tat a failing build provides enough meaningful # information at log-level ERROR. mkdir -p "${OUT}" "${JUST}" build --local-build-root "${LBR}" \ -f "${OUT}/log" --log-limit 0 \ -D '{"foo": "FOO", "irrelevant": "abc", "magic":"xyz"}' \ 2>&1 && exit 1 || : # The exit code should be reported grep 42 "${OUT}/log" # The target name should be reported grep theTargetCausingTheFailure "${OUT}/log" # The pruned effective configuration should be reported in canonical # compact form. grep '{"foo":"FOO","magic":"xyz"}' "${OUT}/log" # At default level we should also find stdout of the target echo "${JUST}" build --local-build-root "${LBR}" \ -f "${OUT}/log.default" \ -D '{"foo": "FOO", "irrelevant": "abc", "magic":"xyz"}' \ 2>&1 && exit 1 || : grep stdout-of-failing-target-FOO-xyz "${OUT}/log.default" # ... as well as command and environment in canonical compact form grep 'echo -n stdout-of-;' "${OUT}/log.default" grep '{"MAGIC_VAR":"xyz"}' "${OUT}/log.default" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/identical-inputs.sh000066400000000000000000000023041516554100600277770ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" LBR="${TEST_TMPDIR}/local-build-root" mkdir sample-repo cd sample-repo touch ROOT cat > TARGETS <<'EOF' { "many files": { "type": "generic" , "out_dirs": ["out"] , "cmds": [ "mkdir -p out" , "for i in $(seq 1 200000) ; do echo same-content > out/$i.txt ; done" ] } , "": { "type": "generic" , "outs": ["all.txt"] , "deps": ["many files"] , "cmds": ["for f in out/*.txt ; do cat $f >> all.txt ; done"] } } EOF "${JUST}" build --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' 2>&1 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/incomplete-retry.sh000066400000000000000000000043261516554100600300330ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="${TMPDIR}/local-build-root" readonly OUT="${TMPDIR}/out" readonly JUST="${ROOT}/bin/tool-under-test" readonly LOCAL_TOOLS="${TMPDIR}/tools" mkdir -p "${LOCAL_TOOLS}" cat > "${LOCAL_TOOLS}/flaky" <<'EOF' echo Will fail to create files, but exit 0 exit 0 EOF chmod 755 "${LOCAL_TOOLS}/flaky" touch ROOT cat > TARGETS <&1 && exit 1 || : cat > "${LOCAL_TOOLS}/flaky" <<'EOF' echo This time will also create the files echo Hello > a.txt echo World > b.txt exit 0 EOF # Retrying should succeed, as semantically failed actions should not # be used from cache. mkdir -p "${OUT}" "${JUST}" install --local-build-root "${LBRDIR}" -o "${OUT}" 2>&1 grep Hello "${OUT}/a.txt" grep World "${OUT}/b.txt" echo echo Remote Testing echo REMOTE_ARGS="--local-build-root ${LBRDIR} -r ${REMOTE_EXECUTION_ADDRESS}" if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" fi cat > "${LOCAL_TOOLS}/flaky" <<'EOF' echo Will fail to create files, but exit 0 exit 0 EOF "${JUST}" build ${REMOTE_ARGS} 2>&1 && exit 1 || : cat > "${LOCAL_TOOLS}/flaky" <<'EOF' echo This time will also create the files echo Hello > a.txt echo World > b.txt exit 0 EOF rm -rf "${OUT}" mkdir -p "${OUT}" "${JUST}" install ${REMOTE_ARGS} -o "${OUT}" 2>&1 grep Hello "${OUT}/a.txt" grep World "${OUT}/b.txt" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/nested-trees.sh000077500000000000000000000061421516554100600271340ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir tool-root mkdir work cd work touch ROOT cat > RULES <<'EOI' { "2^3": { "expression": { "type": "let*" , "bindings": [ ["blob", {"type": "BLOB", "data": "Hello World"}] , [ "two" , { "type": "TREE" , "$1": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "0" , "value": {"type": "var", "name": "blob"} } , { "type": "singleton_map" , "key": "1" , "value": {"type": "var", "name": "blob"} } ] } } ] , [ "four" , { "type": "TREE" , "$1": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "0" , "value": {"type": "var", "name": "two"} } , { "type": "singleton_map" , "key": "1" , "value": {"type": "var", "name": "two"} } ] } } ] , [ "eight" , { "type": "TREE" , "$1": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "0" , "value": {"type": "var", "name": "four"} } , { "type": "singleton_map" , "key": "1" , "value": {"type": "var", "name": "four"} } ] } } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": "data" , "value": {"type": "var", "name": "eight"} } } } } } EOI cat > TARGETS <<'EOI' { "data": {"type": "2^3"} , "ALL": { "type": "generic" , "outs": ["index.txt"] , "cmds": ["find data -type f | sort > index.txt"] , "deps": ["data"] } } EOI echo echo Analysis echo ../bin/tool-under-test analyse data \ --local-build-root ../tool-root --dump-trees ../trees.json --dump-blobs ../blobs.json 2>&1 echo echo Blobs echo cat ../blobs.json [ $(cat ../blobs.json | jq -acM 'length') -eq 1 ] echo echo Trees echo cat ../trees.json [ $(cat ../trees.json | jq -acM 'length') -eq 3 ] echo echo Build echo ../bin/tool-under-test install -L '["env", "PATH='"${PATH}"'"]' \ -o ../out --local-build-root ../tool-root 2>&1 echo echo Index echo cat ../out/index.txt echo [ $(cat ../out/index.txt | wc -l) -eq 8 ] just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/request-action-input.sh000066400000000000000000000037301516554100600306270ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${TMPDIR}/out" mkdir -p "${OUT}" readonly JUST="${ROOT}/bin/tool-under-test" mkdir work cd work touch ROOT cat > RULES <<'EOF' { "complex action": { "expression": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "cmd": ["sh", "-c", "mkdir -p foo && echo bar > foo/out.txt"] , "cwd": "deep/inside" , "outs": ["deep/inside/foo/out.txt"] , "execution properties": {"type": "'", "$1": {"image": "something-fancy", "runner": "special"}} , "timeout scaling": 2 } } } } EOF cat > TARGETS <<'EOF' {"": {"type": "complex action"}} EOF # Sanity check: build succeeds and gives the correct ouput "${JUST}" install --local-build-root "${LBR}" -o "${OUT}" \ -L '["env", "PATH='"${PATH}"'"]' 2>&1 grep bar "${OUT}/deep/inside/foo/out.txt" # Analyse first action and dump provides map "${JUST}" analyse --local-build-root "${LBR}" \ --request-action-input 0 \ --dump-provides "${OUT}/provides.json" 2>&1 [ $(jq '."timeout scaling" | . == 2' "${OUT}/provides.json") = true ] [ $(jq -r '.cwd' "${OUT}/provides.json") = "deep/inside" ] [ $(jq -r '."execution properties".image' "${OUT}/provides.json") = something-fancy ] [ $(jq -r '."execution properties".runner' "${OUT}/provides.json") = special ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/tree-conflicts.sh000066400000000000000000000053571516554100600274570ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" readonly JUST="${ROOT}/bin/tool-under-test" mkdir work cd work touch ROOT cat > TARGETS <<'EOF' { "foo": { "type": "generic" , "out_dirs": ["out"] , "cmds": ["mkdir -p out", "echo FOO > out/foo", "echo FOOBAR > out/foobar"] } , "good bar": { "type": "generic" , "out_dirs": ["out"] , "cmds": ["mkdir -p out", "echo BAR > out/bar", "echo FOOBAR > out/foobar"] } , "bad bar": { "type": "generic" , "out_dirs": ["out"] , "cmds": ["mkdir -p out", "echo BAR > out/bar", "echo foobar > out/foobar"] } , "spurious conflict": { "type": "disjoint_tree_overlay" , "name": "merge" , "deps": ["foo", "good bar"] } , "TheOffendingTarget": { "type": "disjoint_tree_overlay" , "name": "merge" , "deps": ["foo", "bad bar"] } , "real conflict": {"type": "install", "dirs": [["TheOffendingTarget", "."]]} } EOF # In case of a spourious conflict, things should just build "${JUST}" install --local-build-root "${LBR}" -o "${OUT}/spurious" \ -L '["env", "PATH='"${PATH}"'"]' 'spurious conflict' 2>&1 grep FOO "${OUT}/spurious/merge/out/foo" grep BAR "${OUT}/spurious/merge/out/bar" grep FOOBAR "${OUT}/spurious/merge/out/foobar" # In case of a real conflict, analysis should still work "${JUST}" analyse --local-build-root "${LBR}" \ --dump-graph "${OUT}/graph.json" \ --dump-artifacts-to-build "${OUT}/artifacts.json" \ 'real conflict' 2>&1 echo [ "$(jq -r '.merge.type' "${OUT}/artifacts.json")" = TREE_OVERLAY ] OVERLAY_ID=$(jq -r '.merge.data.id' "${OUT}/artifacts.json") [ "$(jq '."tree_overlays"."'"${OVERLAY_ID}"'".trees | length' "${OUT}/graph.json")" -eq 2 ] # Building should fail, however "${JUST}" build --local-build-root "${LBR}" \ -f "${OUT}/log" \ -L '["env", "PATH='"${PATH}"'"]' 'real conflict' 2>&1 && exit 1 || : echo grep 'foobar' "${OUT}/log" # The location of the conflict should be mentioned grep 'TheOffendingTarget' "${OUT}/log" # The actual origin of that tree-overlay # should be reported as well echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/actions/tree-ops.sh000066400000000000000000000052761516554100600262740ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" readonly JUST="${ROOT}/bin/tool-under-test" mkdir work cd work touch ROOT cat > TARGETS <<'EOF' { "early": { "type": "generic" , "out_dirs": ["out"] , "cmds": ["mkdir -p out", "echo FOO > out/foo", "echo early > out/conflict"] } , "late": { "type": "generic" , "out_dirs": ["out"] , "cmds": ["mkdir -p out", "echo BAR > out/bar", "echo late > out/conflict"] } , "overlay": {"type": "tree_overlay", "name": "overlay", "deps": ["early", "late"]} , "empty": {"type": "tree_overlay", "name": "overlay"} } EOF # Sanity check: the trees to be overlayed are build correctly "${JUST}" install --local-build-root "${LBR}" -o "${OUT}/early" \ -L '["env", "PATH='"${PATH}"'"]' early 2>&1 grep FOO "${OUT}/early/out/foo" grep early "${OUT}/early/out/conflict" "${JUST}" install --local-build-root "${LBR}" -o "${OUT}/late" \ -L '["env", "PATH='"${PATH}"'"]' late 2>&1 grep BAR "${OUT}/late/out/bar" grep late "${OUT}/late/out/conflict" # Analysis of the overlay target should show an overlay action of two trees "${JUST}" analyse --local-build-root "${LBR}" \ --dump-graph "${OUT}/graph.json" \ --dump-artifacts-to-build "${OUT}/artifacts.json" \ overlay 2>&1 echo cat "${OUT}/graph.json" echo [ "$(jq -r '.overlay.type' "${OUT}/artifacts.json")" = TREE_OVERLAY ] OVERLAY_ID=$(jq -r '.overlay.data.id' "${OUT}/artifacts.json") [ "$(jq '."tree_overlays"."'"${OVERLAY_ID}"'".trees | length' "${OUT}/graph.json")" -eq 2 ] # Actual test: the overlay is built correctly "${JUST}" install --local-build-root "${LBR}" -o "${OUT}/overlay" \ --log-limit 5 \ -L '["env", "PATH='"${PATH}"'"]' overlay 2>&1 grep FOO "${OUT}/overlay/overlay/out/foo" grep BAR "${OUT}/overlay/overlay/out/bar" grep late "${OUT}/overlay/overlay/out/conflict" # Verify that the building empty overlay succeeds "${JUST}" install --local-build-root "${LBR}" -o "${OUT}/empty" \ --log-limit 5 \ -L '["env", "PATH='"${PATH}"'"]' empty 2>&1 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/build-fails/000077500000000000000000000000001516554100600247235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/build-fails/TARGETS000066400000000000000000000004571516554100600257650ustar00rootroot00000000000000{ "single_fail_dep": { "type": ["@", "rules", "shell/test", "script"] , "name": ["single_fail_dep"] , "test": ["single_fail_dep.sh"] , "deps": [["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["build-fails"] , "deps": ["single_fail_dep"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/build-fails/single_fail_dep.sh000077500000000000000000000030561516554100600303720ustar00rootroot00000000000000#!/bin/bash # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly NUM_DEPS=100 readonly FAIL_DEP=42 create_target() { local ID=$1 if [ $ID = $FAIL_DEP ]; then # failing target echo '{"type": "generic", "cmds": ["false"], "out_dirs": ["out_'$ID'"]}' else # success target echo '{"type": "generic", "cmds": ["echo '$ID' > out_'$ID'/id"], "out_dirs": ["out_'$ID'"]}' fi } mkdir -p .buildroot touch ROOT cat < TARGETS { $(for i in $(seq $NUM_DEPS); do if [ $i = 1 ]; then echo -n ' '; else echo -n ' , '; fi; echo '"dep_'$i'": '$(create_target $i); done) , "main": { "type": "generic" , "cmds": ["cat out_*/id | sort -n > id"] , "outs": ["id"] , "deps": [ $(for i in $(seq $NUM_DEPS); do if [ $i = 1 ]; then echo -n ' '; else echo -n ' , '; fi; echo '"dep_'$i'"'; done) ] } } EOF cat TARGETS echo "main" >&2 bin/tool-under-test build --local-build-root .buildroot main && exit 1 || [ $? = 1 ] echo "done" >&2 exit just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/000077500000000000000000000000001516554100600254035ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/TARGETS000066400000000000000000000031151516554100600264370ustar00rootroot00000000000000{ "generic_out_dirs": { "type": ["@", "rules", "shell/test", "script"] , "name": ["generic_out_dirs"] , "test": ["generic_out_dirs.sh"] , "deps": [["", "tool-under-test"]] } , "generic_sh": { "type": ["@", "rules", "shell/test", "script"] , "name": ["generic_sh"] , "test": ["generic_sh.sh"] , "keep": ["null.json", "empty.json", "custom.json"] , "deps": [["", "tool-under-test"]] } , "generic_conflict": { "type": ["@", "rules", "shell/test", "script"] , "name": ["generic_conflict"] , "test": ["generic_conflict.sh"] , "keep": ["log"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "filegen_config": { "type": ["@", "rules", "shell/test", "script"] , "name": ["filegen_config"] , "test": ["filegen_config.sh"] , "deps": [["", "tool-under-test"]] } , "tree": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tree"] , "test": ["tree.sh"] , "deps": [["", "tool-under-test"]] } , "symlink_config": { "type": ["@", "rules", "shell/test", "script"] , "name": ["symlink_config"] , "test": ["symlink_config.sh"] , "deps": [["", "tool-under-test"]] } , "export_counting": { "type": ["@", "rules", "shell/test", "script"] , "name": ["export_counting"] , "test": ["export_counting.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["built-in-rules"] , "deps": [ "export_counting" , "filegen_config" , "generic_conflict" , "generic_out_dirs" , "generic_sh" , "symlink_config" , "tree" ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/export_counting.sh000066400000000000000000000054201516554100600311670ustar00rootroot00000000000000#!/bin/bash # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TMPDIR}/local-build-root" readonly LOG="${TMPDIR}/log.txt" mkdir work && cd work touch ROOT cat <<'EOF' > repos.json { "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat <<'EOF' > TARGETS { "greeting": { "type": "generic" , "arguments_config": ["NAME"] , "outs": ["greeting.txt"] , "cmds": [ { "type": "join" , "$1": [ { "type": "join_cmd" , "$1": [ "echo" , "Hello" , {"type": "var", "name": "NAME", "default": "world"} ] } , " > greeting.txt" ] } ] } , "exported": {"type": "export", "flexible_config": ["NAME"], "target": "greeting"} , "world": { "type": "configure" , "target": "exported" , "config": { "type": "let*" , "bindings": [["NAME", "World"], ["unrelated", "foo"]] , "body": {"type": "env", "vars": ["NAME", "unrelated"]} } } , "world-2": { "type": "configure" , "target": "exported" , "config": { "type": "let*" , "bindings": [["NAME", "World"], ["unrelated", "bar"]] , "body": {"type": "env", "vars": ["NAME", "unrelated"]} } } , "universe": { "type": "configure" , "target": "exported" , "config": { "type": "let*" , "bindings": [["NAME", "Universe"], ["unrelated", "foo"]] , "body": {"type": "env", "vars": ["NAME", "unrelated"]} } } , "": { "type": "install" , "files": {"world": "world", "world-2": "world-2", "universe": "universe"} } } EOF # The export target is analysed in 3 configuration, but only two # are different in the relevant part of the configuration; so # we should see two uncached export targets. "${JUST_MR}" --just "${JUST}" --local-build-root "${LBR}" --norc \ build -f "${LOG}" --log-limit 4 2>&1 echo grep 'xport.*2 uncached' "${LOG}" echo # After building once, the export targets should be cached. "${JUST_MR}" --just "${JUST}" --local-build-root "${LBR}" --norc \ build -f "${LOG}" --log-limit 4 2>&1 echo grep 'xport.*2 cached' "${LOG}" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/filegen_config.sh000077500000000000000000000026601516554100600307040ustar00rootroot00000000000000#!/bin/bash # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly LBRDIR="$TMPDIR/local-build-root" touch ROOT cat <<'EOF' > TARGETS { "": { "type": "file_gen" , "name": "read-runfiles" , "data": { "type": "join" , "separator": ";" , "$1": {"type": "runfiles", "dep": "uses config"} } , "deps": ["uses config"] } , "uses config": { "type": "file_gen" , "arguments_config": ["NAME"] , "name": {"type": "var", "name": "NAME", "default": "null"} , "data": "irrelevant" } } EOF bin/tool-under-test analyse -D '{"NAME": "here-be-dragons"}' \ --local-build-root "$LBRDIR" --dump-targets targets.json --dump-blobs blobs.json 2>&1 echo echo "Blobs" cat blobs.json [ $(jq '. == ["here-be-dragons"]' blobs.json) = "true" ] echo echo "Targets" cat targets.json [ $(jq '."@"."".""."" == [{"NAME": "here-be-dragons"}]' targets.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/generic_conflict.sh000066400000000000000000000035241516554100600312400ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="${TMPDIR}/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly MAIN="${ROOT}/main-repo" readonly OTHER="${ROOT}/other-repo" mkdir -p "${OTHER}" cd "${OTHER}" echo {} > TARGETS echo other context > data mkdir -p "${MAIN}" cd "${MAIN}" touch ROOT cat > repos.json < data cat > TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out"] , "cmds": ["cat data > out"] , "deps": [["@", "other", "", "data"], ["TREE", null, "."]] } } EOF echo echo local file should analyse fine echo "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBRDIR}" \ analyse data 2>&1 echo echo The default target should detect a staging conflict echo "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBRDIR}" \ analyse -f log --dump-actions - 2>&1 && exit 1 || : echo # Verify that the error message is related grep -i error log grep -i conflict log echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/generic_out_dirs.sh000077500000000000000000000062241516554100600312720ustar00rootroot00000000000000#!/bin/bash # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e touch ROOT mkdir -p lcl-build cat < TARGETS { "gen_out_dirs": { "type": "generic" , "cmds": ["mkdir -p out", "echo foo > out/foo.txt"] , "out_dirs": ["out"] } , "read_out_dirs": { "type": "generic" , "cmds": ["cat out/foo.txt > bar.txt"] , "outs": ["bar.txt"] , "deps": ["gen_out_dirs"] } , "missing_outs_and_out_dirs": {"type": "generic", "cmds": ["mkdir -p out", "echo foo > out/foo.txt"]} , "out_dirs_contains_a_file": {"type": "generic", "cmds": ["echo foo > foo.txt"], "out_dirs": ["foo.txt"]} , "outs_contains_a_dir": { "type": "generic" , "cmds": ["mkdir out", "echo foo > out/foo.txt"] , "outs": ["out"] } , "collision": { "type": "generic" , "cmds": ["touch foo"] , "outs": ["foo"] , "out_dirs": ["foo"] } } EOF echo "read_out_dirs" >&2 bin/tool-under-test build -L '["env", "PATH='"${PATH}"'"]' --local-build-root lcl-build read_out_dirs echo "done" >&2 echo "missing_outs_and_out_dirs" >&2 # at least one of "outs" or "out_dirs" must be declared. we expect to fail during the analysis phase bin/tool-under-test analyse --local-build-root lcl-build --log-limit 0 -f test.log missing_outs_and_out_dirs && exit 1 || : grep 'outs' test.log && grep 'out_dirs' test.log echo "done" >&2 echo "out_dirs_contains_a_file" >&2 # the directories listed in "out_dirs" are created before the cmd is exectuted # so, we expect the shell to complain that it cannot generate file "foo.txt" # because it is a directory. We don't grep the shell error message because it can # varay. # the analysis phase should run fine bin/tool-under-test analyse --local-build-root lcl-build out_dirs_contains_a_file echo "analysis ok" >&2 bin/tool-under-test build -L '["env", "PATH='"${PATH}"'"]' --local-build-root lcl-build out_dirs_contains_a_file && exit 1 || : echo "done" >&2 echo "outs_contains_a_dir" >&2 # we declared an output file "out", which is actually a tree # if we don't creat directory out, the shell will also complain about nonexisting directory "out" # anlysis should run fine bin/tool-under-test analyse --local-build-root lcl-build outs_contains_a_dir echo "analysis ok" >&2 bin/tool-under-test build -L '["env", "PATH='"${PATH}"'"]' --local-build-root lcl-build -f test.log outs_contains_a_dir && exit 1 || : # grep 'ERROR' test.log | grep 'output file' echo "done" >&2 echo "collision" >&2 # we expect an error during the analysis phase because outs and out_dirs must be disjoint bin/tool-under-test analyse --local-build-root lcl-build --log-limit 0 -f test.log collision && exit 1 || : grep 'disjoint' test.log echo "done" >&2 exit just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/generic_sh.sh000066400000000000000000000030501516554100600300430ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="${TMPDIR}/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "arguments_config": ["SHC"] , "cmds": ["first", "second"] , "sh -c": {"type": "var", "name": "SHC"} , "outs": ["some file"] } } EOF "${JUST}" analyse --local-build-root "${LBRDIR}" \ --dump-graph null.json 2>&1 [ $(jq -aM '.actions | .[] | .command == ["sh", "-c", "first\nsecond\n"]' null.json) = true ] "${JUST}" analyse --local-build-root "${LBRDIR}" \ -D '{"SHC": []}' --dump-graph empty.json 2>&1 [ $(jq -aM '.actions | .[] | .command == ["sh", "-c", "first\nsecond\n"]' empty.json) = true ] "${JUST}" analyse --local-build-root "${LBRDIR}" \ -D '{"SHC": ["custom-shell", "--fancy-option"]}' --dump-graph custom.json 2>&1 [ $(jq -aM '.actions | .[] | .command == ["custom-shell", "--fancy-option", "first\nsecond\n"]' custom.json) = true ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/symlink_config.sh000077500000000000000000000027001516554100600307540ustar00rootroot00000000000000#!/bin/bash # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly LBRDIR="$TMPDIR/local-build-root" touch ROOT cat <<'EOF' > TARGETS { "": { "type": "symlink" , "name": "read-runfiles" , "data": { "type": "join" , "separator": ";" , "$1": {"type": "runfiles", "dep": "uses config"} } , "deps": ["uses config"] } , "uses config": { "type": "symlink" , "arguments_config": ["NAME"] , "name": {"type": "var", "name": "NAME", "default": "null"} , "data": "irrelevant" } } EOF bin/tool-under-test analyse -D '{"NAME": "here/../be/../dragons"}' \ --local-build-root "$LBRDIR" --dump-targets targets.json --dump-blobs blobs.json 2>&1 echo echo "Blobs" cat blobs.json [ $(jq '. == ["here/../be/../dragons"]' blobs.json) = "true" ] echo echo "Targets" cat targets.json [ $(jq '."@"."".""."" == [{"NAME": "here/../be/../dragons"}]' targets.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/built-in-rules/tree.sh000077500000000000000000000024711516554100600267050ustar00rootroot00000000000000#!/bin/bash # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly LBRDIR="$TMPDIR/local-build-root" touch ROOT touch a.txt cat <<'EOF' > TARGETS { "": {"type": "tree", "name": "where/to/stage", "deps": ["content"]} , "content": {"type": "install", "files": {"a": "a.txt", "b": "a.txt", "c/d": "a.txt"}} } EOF bin/tool-under-test analyse \ --local-build-root "$LBRDIR" --dump-trees trees.json --dump-artifacts-to-build artifacts.json 2>&1 echo echo Artifacts cat artifacts.json [ $(jq 'keys == ["where/to/stage"]' artifacts.json) = "true" ] tree_id=$(jq '."where/to/stage"."data"."id"' artifacts.json) echo cat trees.json [ $(jq 'keys == ['$tree_id']' trees.json) = "true" ] [ $(jq '.'$tree_id' | keys == ["a", "b", "c/d"]' trees.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/000077500000000000000000000000001516554100600232775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/TARGETS000066400000000000000000000070431516554100600243370ustar00rootroot00000000000000{ "defaults": { "type": ["@", "rules", "shell/test", "script"] , "name": ["defaults"] , "test": ["defaults.sh"] , "deps": [["", "tool-under-test"]] } , "pwd": { "type": ["@", "rules", "shell/test", "script"] , "name": ["pwd"] , "test": ["pwd.sh"] , "deps": [["", "tool-under-test"]] } , "install": { "type": ["@", "rules", "shell/test", "script"] , "name": ["install"] , "test": ["install.sh"] , "deps": [["", "tool-under-test"]] } , "analyse": { "type": ["@", "rules", "shell/test", "script"] , "name": ["analyse"] , "test": ["analyse.sh"] , "deps": [["", "tool-under-test"]] } , "build -P": { "type": ["@", "rules", "shell/test", "script"] , "name": ["build-P"] , "test": ["build-p.sh"] , "deps": [["", "tool-under-test"]] } , "git cas -P": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-cas-P"] , "test": ["git-cas-p.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "git cas fallback": { "type": ["end-to-end", "with remote"] , "name": ["git-cas-fallback"] , "test": ["git-cas-fallback.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "just-mr reporting": { "type": ["@", "rules", "shell/test", "script"] , "name": ["just-mr-reporting"] , "test": ["just-mr-reporting.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["log/warning.txt"] } , "install --archive": { "type": ["@", "rules", "shell/test", "script"] , "name": ["install-archive"] , "test": ["install-archive.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["src.tar", "reconstructed.tar", "fromstdout.tar"] } , "install archived repo": { "type": ["@", "rules", "shell/test", "script"] , "name": ["install-archived-repo"] , "test": ["install-archived-repo.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["src.tar", "reconstructed.tar", "fromstdout.tar"] } , "conflict report": { "type": ["@", "rules", "shell/test", "script"] , "name": ["conflict-report"] , "test": ["conflict-report.sh"] , "deps": [["", "tool-under-test"]] } , "log limit": { "type": ["@", "rules", "shell/test", "script"] , "name": ["log-limit"] , "test": ["log-limit.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "describe": { "type": ["@", "rules", "shell/test", "script"] , "name": ["describe"] , "test": ["describe.sh"] , "deps": [["", "tool-under-test"]] } , "output": { "type": ["@", "rules", "shell/test", "script"] , "name": ["output"] , "test": ["output.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["log"] } , "cas-resolve-special": { "type": ["@", "rules", "shell/test", "script"] , "name": ["cas-resolve-special"] , "test": ["cas-resolve-special.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "stage": ["cli"] , "deps": { "type": "++" , "$1": [ [ "defaults" , "pwd" , "install" , "build -P" , "analyse" , "git cas -P" , "git cas fallback" , "install --archive" , "install archived repo" , "conflict report" , "describe" ] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": ["just-mr reporting", "log limit", "output", "cas-resolve-special"] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/analyse.sh000077500000000000000000000045131516554100600252750ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) BUILDROOT="${TEST_TMPDIR}/build-root" OUT="${TEST_TMPDIR}/out" mkdir -p "${OUT}" mkdir src cd src touch ROOT cat > RULES <<'EOF' { "provides": { "config_vars": ["VIA_CONFIG"] , "expression": { "type": "RESULT" , "provides": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "fixed_json" , "value": {"type": "singleton_map", "key": "foo", "value": "bar"} } , { "type": "singleton_map" , "key": "variable" , "value": {"type": "var", "name": "VIA_CONFIG"} } ] } } } } EOF cat > TARGETS <<'EOF' {"provides": {"type": "provides"}} EOF "${TOOL}" analyse --local-build-root "${BUILDROOT}" \ --dump-result "${OUT}/result.json" 2>&1 cat "${OUT}/result.json" [ "$(jq 'has("artifacts")' "${OUT}/result.json")" = true ] [ "$(jq 'has("provides")' "${OUT}/result.json")" = true ] [ "$(jq 'has("runfiles")' "${OUT}/result.json")" = true ] "${TOOL}" analyse --local-build-root "${BUILDROOT}" \ --dump-provides "${OUT}/plain.json" 2>&1 cat "${OUT}/plain.json" [ "$(jq '. == { "fixed_json": {"foo": "bar"} , "variable": null}' "${OUT}/plain.json")" = true ] "${TOOL}" analyse --local-build-root "${BUILDROOT}" \ -D '{"VIA_CONFIG": {"x": "y"}}' \ --dump-provides "${OUT}/config.json" 2>&1 cat "${OUT}/config.json" [ "$(jq '. == { "fixed_json": {"foo": "bar"} , "variable": {"x": "y"}}' "${OUT}/config.json")" = true ] "${TOOL}" analyse --local-build-root "${BUILDROOT}" \ -D '{"VIA_CONFIG": {"x": "y"}}' \ --dump-provides - > "${OUT}/stdout.json" diff "${OUT}/config.json" "${OUT}/stdout.json" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/build-p.sh000066400000000000000000000040641516554100600251730ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT=$(pwd) JUST=$(realpath ./bin/tool-under-test) BUILDROOT="${TEST_TMPDIR}/build-root" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "out_dirs": ["foo/bar"] , "cmds": [ "mkdir -p foo/bar/baz/greeting" , "echo Hello World > foo/bar/baz/greeting/hello.txt" ] } } EOF "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" --dump-artifacts out.json 2>&1 echo cat out.json # foo/bar is the output artifact, and it is a tree [ $(jq -rM '."foo/bar"."file_type"' out.json) = "t" ] # Therefore, foo is not an output artifact, so requesting -P leaves stdout empty "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -P foo > foo.txt [ -f foo.txt ] && [ -z "$(cat foo.txt)" ] # Requesting foo/bar gives a human-readable description of the tree "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -P foo/bar | grep baz # ... and so does asking for the unique artifact "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -p | grep baz # going deepter into the tree we stil can get human-readable tree descriptions "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -P foo/bar/baz/greeting | grep hello.txt # Files inside the tree can be retrieved "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -P foo/bar/baz/greeting/hello.txt | grep World echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/cas-resolve-special.sh000066400000000000000000000226561516554100600275070ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks if `just add-to-cas` correctly resolves symlinks. ## set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR_SETUP="${TEST_TMPDIR}/local-build-root-setup" readonly LBR_A="${TEST_TMPDIR}/local-build-root-a" readonly LBR_B="${TEST_TMPDIR}/local-build-root-b" readonly LBR_C="${TEST_TMPDIR}/local-build-root-c" readonly LBR_D="${TEST_TMPDIR}/local-build-root-d" readonly LBR_E="${TEST_TMPDIR}/local-build-root-e" readonly LBR_F="${TEST_TMPDIR}/local-build-root-f" readonly LBR_G="${TEST_TMPDIR}/local-build-root-g" readonly LBR_H="${TEST_TMPDIR}/local-build-root-h" readonly LBR_I="${TEST_TMPDIR}/local-build-root-i" readonly LBR_J="${TEST_TMPDIR}/local-build-root-j" readonly LBR_K="${TEST_TMPDIR}/local-build-root-k" readonly UNPACK_DIR="${TEST_TMPDIR}/unpack-dir" readonly TEST_DATA="The content of the data file in foo" readonly TEST_DIRS="bar/baz" readonly DATA_PATH="bar/baz/data.txt" readonly NON_UPWARDS_LINK_PATH="bar/baz/nonupwards" readonly NON_UPWARDS_LINK_TARGET="data.txt" readonly NON_UPWARDS_REWRITTEN_LINK_TARGET="../../../data.txt" readonly UPWARDS_LINK_PATH="bar/upwards" readonly UPWARDS_LINK_TARGET="../bar/baz" readonly INDIRECT_LINK_PATH="bar/indirect" readonly INDIRECT_LINK_TARGET="upwards/data.txt" readonly CYCLE_LINK_1_PATH="bar/cycle_1" readonly CYCLE_LINK_1_TARGET="cycle_2" readonly CYCLE_LINK_2_PATH="bar/cycle_2" readonly CYCLE_LINK_2_TARGET="cycle_1" mkdir -p "${DISTDIR}" mkdir -p log LOGDIR="$(realpath log)" LOG_ARCHIVE_REPO="${LOGDIR}/archive.txt" LOG_FILE_REPO="${LOGDIR}/file.txt" ### # Set up the archives ## ROOT=$(pwd) mkdir -p "foo/${TEST_DIRS}" echo {} > foo/TARGETS echo -n "${TEST_DATA}" > "foo/${DATA_PATH}" # add resolvable non-upwards symlink ln -s "${NON_UPWARDS_LINK_TARGET}" "foo/${NON_UPWARDS_LINK_PATH}" tar cf "${DISTDIR}/foo-1.2.3_non_upwards.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 CONTENT_NON_UPWARDS=$(git hash-object "${DISTDIR}/foo-1.2.3_non_upwards.tar") echo "Foo archive with resolvable non-upwards symlinks has content ${CONTENT_NON_UPWARDS}" # now add also a resolvable upwards symlink ln -s "${UPWARDS_LINK_TARGET}" "foo/${UPWARDS_LINK_PATH}" # ...and a resolvable non-upwards symlink pointing to it ln -s "${INDIRECT_LINK_TARGET}" "foo/${INDIRECT_LINK_PATH}" tar cf "${DISTDIR}/foo-1.2.3_upwards.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 CONTENT_UPWARDS=$(git hash-object "${DISTDIR}/foo-1.2.3_upwards.tar") echo "Foo archive with resolvable upwards symlinks has content ${CONTENT_UPWARDS}" # now add also cycles, to check for failures to resolve ln -s "${CYCLE_LINK_1_TARGET}" "foo/${CYCLE_LINK_1_PATH}" ln -s "${CYCLE_LINK_2_TARGET}" "foo/${CYCLE_LINK_2_PATH}" tar cf "${DISTDIR}/foo-1.2.3_cycle.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 CONTENT_CYCLES=$(git hash-object "${DISTDIR}/foo-1.2.3_cycle.tar") echo "Foo archive with symlink cycles has content ${CONTENT_CYCLES}" ### # Setup sample repository config ## touch ROOT cat > repos.json <&1 EXPECTED=$(jq -r '.repositories.foo_non_upwards_unresolved.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check non-upwards symlinks resolve tree-upwards same as default === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_B}" \ --resolve-special=tree-upwards "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_non_upwards_unresolved.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check non-upwards symlinks resolve tree-all === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_C}" \ --resolve-special=tree-all "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_non_upwards_resolved.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check non-upwards symlinks resolve all same as tree-all === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_D}" \ --resolve-special=tree-all "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_non_upwards_resolved.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done # Check archive with upwards symlinks confined to the tree rm -rf "${UNPACK_DIR}/*" tar xf "${DISTDIR}/foo-1.2.3_upwards.tar" -C "${UNPACK_DIR}" echo === check confined upwards symlinks resolve ignore === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_E}" \ --resolve-special=ignore "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_upwards_ignore_special.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check confined upwards symlinks resolve tree-upwards === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_F}" \ --resolve-special=tree-upwards "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_upwards_resolve_partially.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check confined upwards symlinks resolve tree-all === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_G}" \ --resolve-special=tree-all "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_upwards_resolve_completely.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check confined upwards symlinks resolve all same as tree-all === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_H}" \ --resolve-special=all "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_upwards_resolve_completely.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done echo === check confined upwards symlinks default fail === "${JUST}" add-to-cas --local-build-root "${LBR_I}" "${UNPACK_DIR}/foo" 2>&1 && exit 1 || : echo failed as expected # Check non-confined upwards symlinks # replace non-upwards link to make it upwards pointing to file copy outside # tree; resulting resolved tree should be the same cp "${UNPACK_DIR}/foo/${DATA_PATH}" "${UNPACK_DIR}" rm "${UNPACK_DIR}/foo/${NON_UPWARDS_LINK_PATH}" ln -s "${NON_UPWARDS_REWRITTEN_LINK_TARGET}" "${UNPACK_DIR}/foo/${NON_UPWARDS_LINK_PATH}" echo === check non-confined upwards symlinks resolve all === COMPUTED=$("${JUST}" add-to-cas --local-build-root "${LBR_J}" \ --resolve-special=all "${UNPACK_DIR}/foo") 2>&1 EXPECTED=$(jq -r '.repositories.foo_upwards_resolve_completely.workspace_root[1]' "${CONF}") [ "${COMPUTED}" = "${EXPECTED}" ] echo done # Check that cycles are detected rm -rf "${UNPACK_DIR}/*" tar xf "${DISTDIR}/foo-1.2.3_cycle.tar" -C "${UNPACK_DIR}" echo === check cycle fail === "${JUST}" add-to-cas --local-build-root "${LBR_K}" \ --resolve-special=all "${UNPACK_DIR}/foo" 2>&1 && exit 1 || : echo failed as expected echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/conflict-report.sh000077500000000000000000000125551516554100600267600ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e JUST="$(pwd)/bin/tool-under-test" LBR="${TEST_TMPDIR}/local-build-root" mkdir src cd src touch ROOT cat > RULES <<'EOF' { "provide": { "target_fields": ["deps"] , "expression": { "type": "RESULT" , "provides": { "type": "singleton_map" , "key": "foo" , "value": { "type": "disjoint_map_union" , "msg": "Dependencies may not overlap" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} } } } } } , "use provides": { "target_fields": ["deps"] , "expression": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "msg": "Provided data may not overlap" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "deps"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "foo" } } } } } } EOF cat > TARGETS <<'EOF' { "greeting.txt": { "type": "generic" , "arguments_config": ["NAME"] , "outs": ["greeting.txt"] , "cmds": [ { "type": "join" , "$1": [ { "type": "join_cmd" , "$1": [ "echo" , "Hello" , {"type": "var", "name": "NAME", "default": "unknown user"} ] } , " > greeting.txt" ] } ] } , "world": { "type": "configure" , "target": "greeting.txt" , "config": {"type": "singleton_map", "key": "NAME", "value": "World"} } , "another world": { "type": "configure" , "target": "greeting.txt" , "config": {"type": "singleton_map", "key": "NAME", "value": "World"} } , "universe": { "type": "configure" , "target": "greeting.txt" , "config": {"type": "singleton_map", "key": "NAME", "value": "Universe"} } , "unrelated": { "type": "generic" , "outs": ["foo.txt"] , "cmds": ["echo unrelated > foo.txt"] } , "install-conflict": { "type": "install" , "dirs": [ ["world", "."] , ["unrelated", "."] , ["universe", "."] , ["another world", "."] ] } , "rule-conflict": { "type": "provide" , "deps": ["world", "unrelated", "universe", "another world"] } , "provided world": {"type": "provide", "deps": ["world"]} , "another provided world": {"type": "provide", "deps": ["another world"]} , "provided unrelated": {"type": "provide", "deps": ["unrelated"]} , "provided universe": {"type": "provide", "deps": ["universe"]} , "provided-conflict": { "type": "use provides" , "deps": [ "provided world" , "provided unrelated" , "provided universe" , "another provided world" ] } , "rule-conflict-in-config": { "type": "provide" , "deps": ["world", "unrelated", "universe", "greeting.txt"] } , "simple-rule-conflict-in-config": { "type": "provide" , "deps": ["unrelated", "universe", "greeting.txt"] } } EOF "${JUST}" build --local-build-root "${LBR}" install-conflict \ -f log --log-limit 0 2>&1 && exit 1 || : echo # We expect, in the error message, to see the full-qualified target names # containing the conflicting artifacts, but not the target not containing # this artifact. grep '\["@","","","world"\]' log grep '\["@","","","another world"\]' log grep '\["@","","","universe"\]' log grep '\["@","","","unrelated"\]' log && exit 1 || : echo echo "${JUST}" build --local-build-root "${LBR}" rule-conflict \ -f log --log-limit 0 2>&1 && exit 1 || : echo grep '\["@","","","world"\]' log grep '\["@","","","another world"\]' log grep '\["@","","","universe"\]' log grep '\["@","","","unrelated"\]' log && exit 1 || : echo echo "${JUST}" build --local-build-root "${LBR}" provided-conflict \ -f log --log-limit 0 2>&1 && exit 1 || : echo grep '\["@","","","provided world"\]' log grep '\["@","","","another provided world"\]' log grep '\["@","","","provided universe"\]' log grep '\["@","","","provided unrelated"\]' log && exit 1 || : # Also verify that the non-null part of the effective configuration is # reported. "${JUST}" build --local-build-root "${LBR}" rule-conflict-in-config \ -f log --log-limit 0 2>&1 -D '{"FOO": "bar", "NAME": "World"}' \ && exit 1 || : echo grep -F '[["@","","","world"],{}]' log grep -F '[["@","","","universe"],{}]' log grep -F '[["@","","","greeting.txt"],{"NAME":"World"}]' log grep -F '["@","","","unrelated"]' log && exit 1 || : echo echo "${JUST}" build --local-build-root "${LBR}" simple-rule-conflict-in-config \ -f log --log-limit 0 2>&1 -D '{"FOO": "bar", "NAME": null}' \ && exit 1 || : echo grep -F '[["@","","","universe"],{}]' log grep -F '[["@","","","greeting.txt"],{}]' log grep -F '["@","","","unrelated"]' log && exit 1 || : echo echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/defaults.sh000066400000000000000000000032661516554100600254510ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR=$(realpath out) mkdir src cd src touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"target_file_name": "TARGETS.local"}}} EOF export CONF=$(realpath repos.json) cat > TARGETS.local <<'EOF' {"a": {"type": "file_gen", "name": "a.txt", "data": "A-top-level"}} EOF echo echo === Default target === ${TOOL} build -C $CONF --local-build-root ${BUILDROOT} -Pa.txt | grep top-level echo echo === top-level module found === mkdir foo cd foo cat > TARGETS <<'EOF' {"a": {"type": "file_gen", "name": "a.txt", "data": "WRONG"}} EOF ${TOOL} build -C $CONF --local-build-root ${BUILDROOT} -Pa.txt | grep top-level echo === correct root referece === cat > TARGETS.local <<'EOF' {"a": {"type": "file_gen", "name": "b.txt", "data": "A-local"} , "": {"type": "install", "deps": ["a", ["", "a"]]} } EOF ${TOOL} install -C $CONF --local-build-root ${BUILDROOT} -o ${OUTDIR}/top-ref 2>&1 echo grep top-level ${OUTDIR}/top-ref/a.txt grep local ${OUTDIR}/top-ref/b.txt echo echo === DONE === just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/describe.sh000066400000000000000000000047711516554100600254240ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" LBR="${TEST_TMPDIR}/build-root" OUT="${TEST_TMPDIR}/output" mkdir work cd work touch ROOT cat > TARGETS <<'EOF' { "payload": { "type": "file_gen" , "arguments_config": ["NAME"] , "name": "greeting.txt" , "data": {"type": "join", "$1": ["Hello ", {"type": "var", "name": "NAME"}, "!"]} } , "exported-target": { "type": "export" , "doc": ["Here we export the canonical GREETING text"] , "target": "payload" , "flexible_config": ["NAME"] , "config_doc": {"NAME": ["The RECIPIENT of the greeting"]} } , "": { "type": "configure" , "doc": ["Configure the canonical greeting to set a DEFAULT name"] , "arguments_config": ["NAME"] , "target": "exported-target" , "config": { "type": "singleton_map" , "key": "NAME" , "value": {"type": "var", "name": "NAME", "default": "world"} } } } EOF # Sanity check: the target description actually works "${JUST}" install --local-build-root "${LBR}" -o "${OUT}" 2>&1 grep 'Hello world' "${OUT}/greeting.txt" # Verify the description of the export target "${JUST}" describe --local-build-root "${LBR}" exported-target \ > "${OUT}/export.doc" 2> "${OUT}/log" echo cat "${OUT}/log" echo cat "${OUT}/export.doc" echo echo # - rule name grep '"export"' "${OUT}/export.doc" # - top-level description grep GREETING "${OUT}/export.doc" # - flexible config grep NAME "${OUT}/export.doc" # - documentation of variable grep RECIPIENT "${OUT}/export.doc" # Verify the description of the configure target "${JUST}" describe --local-build-root "${LBR}" \ > "${OUT}/configure.doc" 2> "${OUT}/log" echo cat "${OUT}/log" echo cat "${OUT}/configure.doc" echo echo # - rule name grep '"configure"' "${OUT}/configure.doc" # - top-level description grep DEFAULT "${OUT}/configure.doc" # - the expression defining what to configure grep '"exported-target"' "${OUT}/configure.doc" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/git-cas-fallback.sh000066400000000000000000000035731516554100600267270ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" BUILDROOT="${TEST_TMPDIR}/build-root" OUT="${TEST_TMPDIR}/out" WRKDIR="${ROOT}/work" SRCDIR="${ROOT}/srcs" # Add a src git repo mkdir -p "${SRCDIR}" cd "${SRCDIR}" mkdir -p src/with/some/deep/directory echo 'Hello World' > src/with/some/deep/directory/hello.txt # Set up a build root from git repo mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.json < CONF echo cat CONF echo cat $(cat CONF) echo TREE=$(jq -rM '.repositories."".workspace_root[1]' $(cat CONF)) echo echo Tree is ${TREE} echo # Clear local cas, while keeping repositories (and hence git cas) "${JUST}" gc --local-build-root "${BUILDROOT}" 2>&1 "${JUST}" gc --local-build-root "${BUILDROOT}" 2>&1 # Verify that we can access the tree (via the git cas), # even if a remote endpoint is provided. "${JUST}" install-cas --local-build-root "${BUILDROOT}" \ -r "${REMOTE_EXECUTION_ADDRESS}" "${TREE}::t" -o "${OUT}" 2>&1 grep World "${OUT}/src/with/some/deep/directory/hello.txt" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/git-cas-p.sh000066400000000000000000000043101516554100600254150ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" BUILDROOT="${TEST_TMPDIR}/build-root" DISTDIR="${TEST_TMPDIR}/distfiles" mkdir -p "${DISTDIR}" # Create an archive mkdir -p src/with/some/deep/directory echo 'Hello World' > src/with/some/deep/directory/hello.txt tar cvf "${DISTDIR}/src.tar" src rm -rf src HASH=$(git hash-object "${DISTDIR}/src.tar") echo echo archive has content $HASH echo # Set up a build root from that archive mkdir work cd work touch ROOT cat > repos.json < CONF echo cat CONF echo cat $(cat CONF) echo TREE=$(jq -rM '.repositories."".workspace_root[1]' $(cat CONF)) echo echo Tree is ${TREE} echo # Verify that we can obtain this tree, as well as its parts "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" -P some "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" -P some/deep "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" -P some/deep/directory "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" -P some/deep/directory/hello.txt "${JUST}" install-cas --local-build-root "${BUILDROOT}" "${TREE}::t" -P some/deep/directory/hello.txt -o hello.txt grep World hello.txt echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/install-archive.sh000066400000000000000000000061731516554100600267270ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ROOT=$(pwd) JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" BUILD_ROOT_A="${TEST_TMPDIR}/build-root-A" BUILD_ROOT_B="${TEST_TMPDIR}/build-root-B" BUILD_ROOT_C="${TEST_TMPDIR}/build-root-C" mkdir -p src echo simple file > src/foo.txt (cd src && ln -s foo symlink) mkdir -p src/foo/bar echo another file > src/foo/bar/data.txt mkdir -p src/empty # empty dir echo TREE=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_A}" src) echo "Source tree is ${TREE}" echo # Generate archive and verify it has the correct content "${JUST}" install-cas --local-build-root "${BUILD_ROOT_A}" \ --archive -o src.tar "${TREE}::t" 2>&1 mkdir src.unpacked (cd src.unpacked && tar xvf ../src.tar 2>&1) diff -ruN src src.unpacked # clean up no longer needed directories rm -rf src src.unpacked # On a new build root, add the archive ARCHIVE=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_B}" src.tar) echo echo Archive has content "${ARCHIVE}" echo # Verify that we can create the original tree from that archive mkdir work cd work touch ROOT cat > repos.json < TARGETS <<'EOF' {"": {"type": "install", "dirs": [[["TREE", null, "."], "."]]}} EOF CONF="$("${JUST_MR}" --norc --local-build-root "${BUILD_ROOT_B}" setup)" echo echo configuration $CONF cat $CONF echo RECONSTRUCTED_TREE=$(jq -r '.repositories."".workspace_root[1]' "${CONF}") [ "${RECONSTRUCTED_TREE}" = "${TREE}" ] # Build to get the tree unconditionally known to the local build root "${JUST}" build --local-build-root "${BUILD_ROOT_B}" -C "${CONF}" 2>&1 # - installing the tree as archive should give the same file "${JUST}" install-cas --local-build-root "${BUILD_ROOT_B}" \ -o "${ROOT}/reconstructed.tar" --archive "${TREE}::t" 2>&1 cmp "${ROOT}/src.tar" "${ROOT}/reconstructed.tar" 2>&1 # - dumping the tree as archive to stdout also gives the same archive "${JUST}" install-cas --local-build-root "${BUILD_ROOT_B}" \ --archive "${TREE}::t" > "${ROOT}/fromstdout.tar" cmp "${ROOT}/src.tar" "${ROOT}/fromstdout.tar" 2>&1 # - hashing the archive again gives the original value ARCHIVE_NEWLY_HASHED=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_C}" "${ROOT}/fromstdout.tar") [ "${ARCHIVE_NEWLY_HASHED}" = "${ARCHIVE}" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/install-archived-repo.sh000066400000000000000000000074411516554100600300350ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # Git repositories treat some filenames, such as .git, .gitignore etc., as # magic names. We should not have such restrictions in handling blobs and trees. # # This test checks that trees can be added to CAS as they are, without # ignoring/skipping any files or folders that might have special meaning to Git. ## set -eu ROOT=$(pwd) JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" BUILD_ROOT_A="${TEST_TMPDIR}/build-root-A" BUILD_ROOT_B="${TEST_TMPDIR}/build-root-B" BUILD_ROOT_C="${TEST_TMPDIR}/build-root-C" # create a Git repo, including a .gitignore file mkdir -p repo echo simple file > repo/foo.txt (cd repo && ln -s foo symlink) mkdir -p repo/foo/bar echo another file > repo/foo/bar/data.txt ( cd repo echo data.txt > foo/bar/.gitignore git init git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add -f . 2>&1 git commit -m "initial commit" ) find repo ! -name . -printf "%P\n" tar cf repo.tar repo # Add repo directory to CAS and print tree id as we see it (with all entries) echo TREE=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_A}" repo) echo "Source tree is ${TREE}" echo # Generate archive and verify it has the correct content "${JUST}" install-cas --local-build-root "${BUILD_ROOT_A}" \ --archive -o repo.tar "${TREE}::t" 2>&1 mkdir repo.unpacked (cd repo.unpacked && tar xvf ../repo.tar 2>&1) diff -ruN repo repo.unpacked # clean up no longer needed directories rm -rf repo.unpacked # On a new build root, add the archive ARCHIVE=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_B}" repo.tar) echo echo Archive has content "${ARCHIVE}" echo # Verify that we can create the original tree from that archive mkdir work cd work touch ROOT cat > repos.json < TARGETS <<'EOF' {"": {"type": "install", "dirs": [[["TREE", null, "."], "."]]}} EOF CONF="$("${JUST_MR}" --norc --local-build-root "${BUILD_ROOT_B}" setup)" echo echo configuration $CONF cat $CONF echo RECONSTRUCTED_TREE=$(jq -r '.repositories."".workspace_root[1]' "${CONF}") [ "${RECONSTRUCTED_TREE}" = "${TREE}" ] # Build to get the tree unconditionally known to the local build root "${JUST}" build --local-build-root "${BUILD_ROOT_B}" -C "${CONF}" 2>&1 # - installing the tree as archive should give the same content "${JUST}" install-cas --local-build-root "${BUILD_ROOT_B}" \ -o "${ROOT}/reconstructed.tar" --archive "${TREE}::t" 2>&1 cmp "${ROOT}/repo.tar" "${ROOT}/reconstructed.tar" 2>&1 # - dumping the tree as archive to stdout also gives the same archive "${JUST}" install-cas --local-build-root "${BUILD_ROOT_B}" \ --archive "${TREE}::t" > "${ROOT}/fromstdout.tar" cmp "${ROOT}/repo.tar" "${ROOT}/fromstdout.tar" 2>&1 # - hashing the archive again gives the original value ARCHIVE_NEWLY_HASHED=$("${JUST}" add-to-cas --local-build-root "${BUILD_ROOT_C}" "${ROOT}/fromstdout.tar") [ "${ARCHIVE_NEWLY_HASHED}" = "${ARCHIVE}" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/install.sh000066400000000000000000000061401516554100600253020ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT=$(pwd) TOOL=$(realpath ./bin/tool-under-test) BUILDROOT="${TEST_TMPDIR}/build-root" mkdir src touch src/ROOT cat > src/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["hello.txt"] , "cmds": ["echo Hello World > hello.txt"] } , "symlink": { "type": "generic" , "outs": ["hello.txt", "content.txt"] , "cmds": ["echo Hello World > content.txt", "ln -s content.txt hello.txt"] } } EOF SRCDIR=$(realpath src) mkdir out echo 'Original' > out/unrelated.txt ln out/unrelated.txt out/hello.txt OUTDIR=$(realpath out) ls -al "${OUTDIR}" echo # Verify non-interference of install cd "${SRCDIR}" "${TOOL}" install -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -o "${OUTDIR}" 2>&1 echo ls -al "${OUTDIR}" cd "${OUTDIR}" grep World hello.txt grep Original unrelated.txt # Verify non-interference of install cd "${SRCDIR}" "${TOOL}" build -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" \ --dump-artifacts "${OUTDIR}/artifacts.json" 2>&1 echo ID=$(jq -rM '."hello.txt".id' "${OUTDIR}/artifacts.json") ln "${OUTDIR}/unrelated.txt" "${OUTDIR}/${ID}" ls -al "${OUTDIR}" echo "${TOOL}" install-cas --local-build-root "${BUILDROOT}" \ -o "${OUTDIR}" "${ID}" 2>&1 echo ls -al "${OUTDIR}" cd "${OUTDIR}" grep World "${ID}" grep Original unrelated.txt # Verify non-interference of install (overwrite existing file with symlink) cd "${SRCDIR}" "${TOOL}" install -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -o "${OUTDIR}" symlink 2>&1 echo ls -al "${OUTDIR}" cd "${OUTDIR}" grep World hello.txt grep Original unrelated.txt [ "$(realpath --relative-to=$(pwd) hello.txt)" = "content.txt" ] # Verify non-interference of install (overwrite existing symlink with symlink) rm -f ${OUTDIR}/hello.txt ln -s /noexistent ${OUTDIR}/hello.txt cd "${SRCDIR}" "${TOOL}" install -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -o "${OUTDIR}" symlink 2>&1 echo ls -al "${OUTDIR}" cd "${OUTDIR}" grep World hello.txt grep Original unrelated.txt [ "$(realpath --relative-to=$(pwd) hello.txt)" = "content.txt" ] # Verify non-interference of install (overwrite existing symlink with file) rm -f ${OUTDIR}/content.txt ln -s /noexistent ${OUTDIR}/content.txt cd "${SRCDIR}" "${TOOL}" install -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${BUILDROOT}" -o "${OUTDIR}" symlink 2>&1 echo ls -al "${OUTDIR}" cd "${OUTDIR}" grep World hello.txt grep Original unrelated.txt [ "$(realpath --relative-to=$(pwd) hello.txt)" = "content.txt" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/just-mr-reporting.sh000077500000000000000000000034401516554100600272470ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e BUILDROOT="${TEST_TMPDIR}/build-root" OUTDIR="${TEST_TMPDIR}/out" mkdir tools ln -s $(realpath ./bin/tool-under-test) tools/the-build-tool ln -s $(realpath ./bin/mr-tool-under-test) tools/the-multi-repo-tool export PATH=$(realpath tools):${PATH} mkdir -p log LOGDIR="$(realpath log)" mkdir work cd work touch ROOT cat > repos.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat > TARGETS <<'EOF' { "generated": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo Hello WoRlD > out.txt"] } , "": {"type": "export", "target": "generated"} } EOF the-multi-repo-tool --norc --just the-build-tool \ --local-build-root "${BUILDROOT}" \ --log-limit 1 -f "${LOGDIR}/warning.txt" \ install -o "${OUTDIR}" 2>&1 # Verify that the build yields the expected result grep WoRlD "${OUTDIR}/out.txt" # Verify that the reported warning contains the actual # tool names, not some hardcoded string grep the-build-tool "${LOGDIR}/warning.txt" grep the-multi-repo-tool "${LOGDIR}/warning.txt" grep 'just build' "${LOGDIR}/warning.txt" && exit 1 grep 'just-mr' "${LOGDIR}/warning.txt" && exit 1 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/log-limit.sh000077500000000000000000000130211516554100600255300ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" LBR="${TEST_TMPDIR}/build-root" mkdir work cd work touch ROOT cat > RULES <<'EOF' { "test": { "tainted": ["test"] , "string_fields": ["cmds"] , "expression": { "type": "let*" , "bindings": [ [ "cmd" , { "type": "join" , "separator": "\n" , "$1": {"type": "FIELD", "name": "cmds"} } ] , [ "shell cmd" , { "type": "join_cmd" , "$1": ["sh", "-c", {"type": "var", "name": "cmd"}] } ] , [ "test cmd" , [ "sh" , "-c" , { "type": "join" , "separator": " " , "$1": [ "echo FAIL > RESULT;" , {"type": "var", "name": "shell cmd"} , " && echo PASS > RESULT;" , "if [ $(cat RESULT) != PASS ] ; then exit 1 ; fi" ] } ] ] , [ "result" , { "type": "ACTION" , "cmd": {"type": "var", "name": "test cmd"} , "outs": ["RESULT"] , "may_fail": ["test"] , "fail_message": "TeStFaIlUrE" } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "result"}} } } } EOF cat > TARGETS <<'EOF' { "verbose": { "type": "generic" , "outs": ["out.txt"] , "cmds": [ "echo -n Running the VeR" , "echo bOsE command now" , "echo Hello WoRlD > out.txt" ] } , "test": {"type": "test", "cmds": ["echo TeStRuN", "false"]} , "": { "type": "install" , "tainted": ["test"] , "files": {"verbose": "verbose", "test": "test"} } } EOF echo FAIL > "${TEST_TMPDIR}/result" FAIL_HASH=$("${JUST}" add-to-cas --local-build-root "${LBR}" "${TEST_TMPDIR}/result") "${JUST}" build --local-build-root "${LBR}" -f file.log \ -L '["env", "PATH='"${PATH}"'"]' \ --restrict-stderr-log-limit 1 2>console.log || : echo echo Console cat console.log echo echo Log file cat file.log echo echo verifying log limits ... echo # stdout/stderr of actions are reported at INFO level (included in the default, # but lower than WARN, which the console is restricted to. grep VeRbOsE file.log echo grep VeRbOsE console.log && exit 1 || : echo grep TeStRuN file.log echo grep TeStRuN console.log && exit 1 || : # Failed actions are reported at WARN level, hence should be present in both echo grep TeStFaIlUrE file.log grep TeStFaIlUrE console.log # Also, their outputs should be reported grep "RESULT.*${FAIL_HASH}" file.log grep "RESULT.*${FAIL_HASH}" console.log echo echo echo Same test using the launcher echo touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF rm -f file.log console.log "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" -f file.log \ -L '["env", "PATH='"${PATH}"'"]' \ --restrict-stderr-log-limit 1 build 2>console.log || : echo echo Console cat console.log echo echo Log file cat file.log echo echo verifying log limits ... echo # The information of the launch invocation is logged at INFO level, hence should # be visible in the file log, but not in the console log grep tool-under-test file.log echo grep tool-under-test console.log && exit 1 || : echo # stdout/stderr of actions are reported at INFO level (included in the default, # but lower than WARN, which the console is restricted to. grep VeRbOsE file.log echo grep VeRbOsE console.log && exit 1 || : echo grep TeStRuN file.log echo grep TeStRuN console.log && exit 1 || : # Failed actions are reported at WARN level, hence should be present in both echo grep TeStFaIlUrE file.log grep TeStFaIlUrE console.log echo echo echo Same test using the launcher, with information taken from rc file echo cat > rc.json <console.log || : echo echo Console cat console.log echo echo Log file cat file.log echo echo verifying log limits ... echo # The information of the launch invocation is logged at INFO level, hence should # be visible in the file log, but not in the console log grep tool-under-test file.log echo grep tool-under-test console.log && exit 1 || : echo # stdout/stderr of actions are reported at INFO level (included in the default, # but lower than WARN, which the console is restricted to. grep VeRbOsE file.log echo grep VeRbOsE console.log && exit 1 || : echo grep TeStRuN file.log echo grep TeStRuN console.log && exit 1 || : # Failed actions are reported at WARN level, hence should be present in both echo grep TeStFaIlUrE file.log grep TeStFaIlUrE console.log # Also, their outputs should be reported grep "RESULT.*${FAIL_HASH}" file.log grep "RESULT.*${FAIL_HASH}" console.log echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/output.sh000066400000000000000000000032261516554100600251760ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ROOT="$(pwd)" JUST="${ROOT}/bin/tool-under-test" JUST_MR="${ROOT}/bin/mr-tool-under-test" LBR="${TEST_TMPDIR}/build-root" LOG="${ROOT}/log" mkdir work cd work touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "generic": { "type": "generic" , "outs": ["hello.txt"] , "cmds": ["echo Hello World > hello.txt"] } , "": {"type": "install", "files": {"generic": "generic"}} } EOF ${JUST_MR} --norc -L '["env", "PATH='"${PATH}"'"]' --local-build-root "${LBR}" \ --just "${JUST}" \ -f "${LOG}" build 2>&1 echo # Sanity check on verbosity of output ## By default, we should be at a log level reporting resulting artifacts HELLO_BLOB_ID=$(echo Hello World | git hash-object --stdin) grep -i $HELLO_BLOB_ID "${LOG}" ## This build should not cause any errors or warnings grep -i error "${LOG}" && exit 1 || : grep -i warn "${LOG}" && exit 1 || : ## Such a simple build should not have an overly long log [ $(cat "${LOG}" | wc -l) -lt 100 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/cli/pwd.sh000066400000000000000000000041671516554100600244350ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT=$(pwd) TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR=$(realpath out) mkdir src touch src/ROOT cat > src/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["hello.txt"] , "cmds": ["echo Hello World > hello.txt"] } } EOF SRCDIR=$(realpath src) cat > repos.json < worker.sh < "${WORKER_LOG}" 2>&1 touch "${DONE_FILE}" EOF chmod 755 worker.sh echo cat worker.sh echo mkdir -p "${WORK_DIR}" echo Created "${WORK_DIR}", starting worker (./worker.sh) & echo Wating for the worker to enter its work dir while [ ! -f "${ENTERED_DIR_FILE}" ] do sleep 1 done echo Removing work dir rm -rf "${WORK_DIR}" touch "${WORK_DIR_REMOVED_FILE}" echo Waiting for the worker to finish while [ ! -f "${DONE_FILE}" ] do sleep 1 done echo Worker finished, output was cat "${WORKER_LOG}" echo echo Inspecting result ls "${OUTDIR}" grep World "${OUTDIR}/hello.txt" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/000077500000000000000000000000001516554100600255145ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/TARGETS000066400000000000000000000065601516554100600265570ustar00rootroot00000000000000{ "basic": { "type": ["@", "rules", "shell/test", "script"] , "name": ["basic"] , "test": ["basic.sh"] , "deps": [["", "tool-under-test"]] , "keep": [ "out/base/TARGETS" , "out/derived/out" , "out/other-derived/out" , "out/log" , "out/log.root" , "out/log2" ] } , "build params": { "type": ["@", "rules", "shell/test", "script"] , "name": ["build-params"] , "test": ["build-params.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/stdout", "out/stderr"] } , "mr_computed_setup": { "type": ["@", "rules", "shell/test", "script"] , "name": ["mr_computed_setup"] , "test": ["mr_computed_setup.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "mr_take_over": { "type": ["@", "rules", "shell/test", "script"] , "name": ["mr_take_over"] , "test": ["mr_take_over.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "error-reporting": { "type": ["@", "rules", "shell/test", "script"] , "name": ["error-reporting"] , "test": ["error-reporting.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/not-export.log", "out/not-content-fixed.log", "out/cycle.log"] } , "error-remote": { "type": ["end-to-end", "with remote"] , "name": ["error-remote"] , "test": ["error-remote.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/build.log"] } , "basic remote build": { "type": ["end-to-end", "with remote"] , "name": ["remote"] , "test": ["remote.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/log", "out/log.root"] } , "remote sharding": { "type": ["end-to-end", "with remote"] , "name": ["sharding"] , "test": ["sharding.sh"] , "deps": [["", "tool-under-test"]] , "keep": [ "out/base-local.log" , "out/derived-local.log" , "out/derived-local/out" , "out/derived-local2.log" , "out/derived-remote.log" , "out/derived-remote/out" ] } , "artifacts-only": { "type": ["end-to-end", "with remote"] , "name": ["artifacts-only"] , "test": ["artifacts-only.sh"] , "deps": [["", "tool-under-test"]] } , "absent base of root (data)": {"type": "install", "files": {"TARGETS": "targets.absent-base"}} , "absent base of root": { "type": ["end-to-end", "with serve"] , "name": ["absent-base"] , "test": ["absent-base.sh"] , "deps": [["", "tool-under-test"]] , "repos": ["absent base of root (data)"] } , "absent target root (data)": { "type": "install" , "files": { "TARGETS": "targets.absent-targets" , "data/TARGETS": "targets.absent-targets-data" } } , "absent computed root": { "type": ["end-to-end", "with serve"] , "name": ["absent-computed"] , "test": ["absent-computed.sh"] , "deps": [["", "tool-under-test"]] , "repos": ["absent base of root (data)", "absent target root (data)"] , "keep": ["out/build.log"] } , "progress": { "type": ["@", "rules", "shell/test", "script"] , "name": ["progress"] , "test": ["progress.sh"] , "deps": [["", "tool-under-test"]] , "keep": ["out/log"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["computed-roots"] , "deps": [ "absent base of root" , "absent computed root" , "artifacts-only" , "basic" , "basic remote build" , "build params" , "error-remote" , "error-reporting" , "mr_computed_setup" , "mr_take_over" , "progress" , "remote sharding" ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/absent-base.sh000066400000000000000000000034421516554100600302370ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly ROOT="${PWD}" readonly JUST="${ROOT}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly MAIN="${ROOT}/main" readonly OUT="${TEST_TMPDIR}/out" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir -p "${MAIN}" cd "${MAIN}" cat > repo-config.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out"] , "cmds": ["cat *.txt > out"] , "deps": [["data", ""]] } } EOF mkdir -p targets/data cat > targets/data/TARGETS <<'EOF' {"": {"type": "install", "deps": [["GLOB", null, "*.txt"]]}} EOF echo echo Build against computed root with absent base echo "${JUST}" install -o "${OUT}" -C repo-config.json \ --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --log-limit 4 2>&1 # sanity check output [ $(cat "${OUT}/out" | wc -l) -eq 55 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/absent-computed.sh000066400000000000000000000032361516554100600311460ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly ROOT="${PWD}" readonly JUST="${ROOT}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly MAIN="${ROOT}/main" readonly TARGETS="${ROOT}/targets" readonly OUT="${TEST_TMPDIR}/out" readonly LOG="${TEST_TMPDIR}/overall.log" readonly BUILD_LOG="${OUT}/build.log" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir -p "${MAIN}" cd "${MAIN}" cat > repo-config.json <&1 # sanity check output [ $(cat "${OUT}/out" | wc -l) -eq 55 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/artifacts-only.sh000066400000000000000000000077021516554100600310150ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="${TMPDIR}/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${TMPDIR}/out" readonly OUT2="${TMPDIR}/out2" # The export target "" of the base root has # - artifacts: TARGETS, artifacts.txt # - runfiles: runfiles.txt # So the glob in the TARGETS file of a computed root based # on that export target depends on whether the runfiles are # taken as part of the root (which is wrong) or not. readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > TARGETS <<'EOF' { "": {"type": "export", "target": "root"} , "root": {"type": "root", "targets": ["payload_targets"]} } EOF cat > RULES <<'EOF' { "root": { "target_fields": ["targets"] , "expression": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": [ { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "targets"} , "body": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "range": { "type": "values" , "$1": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "_"} } } , "body": { "type": "`" , "$1": { "TARGETS": {"type": ",", "$1": {"type": "var", "name": "_"}} } } } } } } , { "type": "`" , "$1": { "artifact.txt": {"type": ",", "$1": {"type": "BLOB", "data": "ArTiFaCt"}} } } ] } , "runfiles": { "type": "`" , "$1": { "runfile.txt": {"type": ",", "$1": {"type": "BLOB", "data": "RUNFILE!"}} } } } } } EOF cat > payload_targets <<'EOF' { "": { "type": "generic" , "deps": [["GLOB", null, "*.txt"]] , "outs": ["out"] , "cmds": ["cat $(ls *.txt |sort) > out"] } } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo echo Build on the computed root echo "${JUST}" install --local-build-root "${LBR}" -L '["env", "PATH='"${PATH}"'"]' \ -C repo-config.json -o "${OUT}" 2>&1 echo cat "${OUT}/out" echo grep ArTiFaCt "${OUT}/out" grep RUNFILE "${OUT}/out" && exit 1 || : echo echo Build on the computed root again, to verify same for a cached root echo "${JUST}" install --local-build-root "${LBR}" -L '["env", "PATH='"${PATH}"'"]' \ -C repo-config.json -o "${OUT2}" 2>&1 echo cat "${OUT2}/out" echo grep ArTiFaCt "${OUT2}/out" grep RUNFILE "${OUT2}/out" && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/basic.sh000066400000000000000000000076601516554100600271420ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } ] } } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo cat "${OUT}/base/TARGETS" echo echo Building computed echo "${JUST}" install -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 --main derived -o "${OUT}/derived" 2>&1 echo [ "$(cat "${OUT}/derived/out" | wc -l)" -eq 55 ] echo echo Building a different computed root, without reference build echo "${JUST}" install -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 -f "${OUT}/log" \ --main 'other derived' -o "${OUT}/other-derived" 2>&1 echo [ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ] echo echo Sanity-check of the log echo grep '[Rr]oot.*base.*evaluated.*' "${OUT}/log" > "${TMPDIR}/log_line" cat "${TMPDIR}/log_line" sed -i 's/.*log //' "${TMPDIR}/log_line" "${JUST}" install-cas --local-build-root "${LBRDIR}" \ -o "${OUT}/log.root" $(cat "${TMPDIR}/log_line") echo cat "${OUT}/log.root" echo grep 'COUNT.*12' "${OUT}/log.root" grep '[Dd]iscovered.*1 action' "${OUT}/log.root" grep '0 cache hit' "${OUT}/log.root" echo echo Building computed root again, expecting target-level cache hit echo "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 -f "${OUT}/log2" \ --main 'other derived' 2>&1 echo grep '[Rr]oot.*from cache' "${OUT}/log2" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/build-params.sh000066400000000000000000000035271516554100600304370ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > TARGETS <<'EOF' { "": {"type": "export", "target": "root"} , "root": { "type": "install" , "files": {"TARGETS": "payload-target", "out": "unrelated"} } } EOF cat > payload-target <<'EOF' {"": {"type": "generic", "outs": ["out"], "cmds": ["echo GoodOutput > out"]}} EOF echo DoNOTLookAtMe > unrelated git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json < "${OUT}/stdout" 2> "${OUT}/stderr" grep GoodOutput "${OUT}/stdout" grep DoNOT "${OUT}/stdout" && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/error-remote.sh000066400000000000000000000045011516554100600304720ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRA="$TMPDIR/local-build-root-A" readonly LBRB="$TMPDIR/local-build-root-B" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" REMOTE_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" fi echo Using ${REMOTE_ARGS} readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > TARGETS <<'EOF' { "": {"type": "export", "target": "failing build"} , "failing build": { "type": "generic" , "outs": ["TARGETS"] , "cmds": ["echo -n ThisWill", "echo FAIL", "false"] } } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 && exit 1 || : LOG_BLOB=$(grep 'see [a-zA-Z0-9]* for details' "${OUT}/build.log" | sed 's/.*see //' | sed 's/ for details.*//') echo echo "Blob of log is ${LOG_BLOB}" echo echo 'Verifying that the blob can be downloaded from a different client' echo "${JUST}" install-cas ${REMOTE_ARGS} \ --local-build-root "${LBRB}" \ -o "${TMPDIR}/log" "${LOG_BLOB}" 2>&1 echo grep ThisWillFAIL "${TMPDIR}/log" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/error-reporting.sh000066400000000000000000000076621516554100600312230ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } ] } } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo [ "$(cat "${OUT}/derived/out" | wc -l)" -eq 55 ] echo echo Not an export echo "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ -f "${OUT}/not-export.log" \ --log-limit 4 --main 'not export' 2>&1 && exit 1 || : echo grep '[Tt]arget.*not.*export' "${OUT}/not-export.log" echo echo Not content fixed echo "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ -f "${OUT}/not-content-fixed.log" \ --log-limit 4 --main 'not content-fixed' 2>&1 && exit 1 || : echo grep 'Repository.*file base.*not.*content.*fixed' "${OUT}/not-content-fixed.log" echo echo cycle echo "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ -f "${OUT}/cycle.log" \ --log-limit 4 --main 'cycle-A' 2>&1 && exit 1 || : echo grep cycle-A "${OUT}/cycle.log" grep cycle-B "${OUT}/cycle.log" grep cycle-C "${OUT}/cycle.log" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/mr_computed_setup.sh000066400000000000000000000075771516554100600316260ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly OUT="${ROOT}/out" readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } ] } } EOF mkdir -p "${ROOT}/main" cd "${ROOT}/main" touch ROOT cat > repo-config.json <&1 echo cat "${OUT}/base/TARGETS" echo echo Building computed echo "${JUST_MR}" --norc --local-build-root "${LBRDIR}" -C repo-config.json \ --main derived --just "${JUST}" \ install -L '["env", "PATH='"${PATH}"'"]' -o "${OUT}/derived" 2>&1 echo [ "$(cat "${OUT}/derived/out" | wc -l)" -eq 55 ] echo echo Building a different computed root, without reference build echo "${JUST_MR}" --norc --local-build-root "${LBRDIR}" -C repo-config.json \ --main 'other derived' --just "${JUST}" \ install -L '["env", "PATH='"${PATH}"'"]' \ -o "${OUT}/other-derived" 2>&1 echo [ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ] echo echo Building with overlays echo "${JUST_MR}" --norc --local-build-root "${LBRDIR}" -C repo-config.json \ --main 'with_overlays' --just "${JUST}" \ install -L '["env", "PATH='"${PATH}"'"]' \ -o "${OUT}/with-overlays" 2>&1 echo [ "$(cat "${OUT}/with-overlays/out" | wc -l)" -eq 55 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/mr_take_over.sh000066400000000000000000000061001516554100600305220ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly OUT="${ROOT}/out" readonly SRC_DIR="${ROOT}/src" mkdir -p "${SRC_DIR}" cat > "${SRC_DIR}/generate.py" <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF readonly TARGETS_DIR="${ROOT}/targets" mkdir -p "${TARGETS_DIR}" cd "${TARGETS_DIR}" cat > "TARGETS.collect" << EOF { "": {"type": "export", "target": "collect"} , "collect": { "type": "install" , "deps": [["TREE", null, "."]] } } EOF cat > TARGETS.compute <<'EOF' { "": {"type": "export", "target": "generate"} , "generate": { "type": "generic" , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , "10" , ">" , "TARGETS" ] } ] } } EOF mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/progress.sh000066400000000000000000000070221516554100600277150ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > generate.py <<'EOF' # A deliberately slow action for generating a root import json import sys import time COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): time.sleep(1) targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } ] } } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json < TARGETS <<'EOF' { "": { "type": "install" , "dirs": [ [["@", "a", "", ""], "a"] , [["@", "b", "", ""], "b"] , [["@", "c", "", ""], "c"] , [["@", "d", "", ""], "d"] ] } } EOF echo echo Building echo "${JUST}" install -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ -f "${OUT}/log" -o "${OUT}/out" 2>&1 echo # Sanity check for build [ "$(cat "${OUT}/out/a/out" | wc -l)" -eq 55 ] [ "$(cat "${OUT}/out/b/out" | wc -l)" -eq 66 ] [ "$(cat "${OUT}/out/c/out" | wc -l)" -eq 78 ] [ "$(cat "${OUT}/out/d/out" | wc -l)" -eq 91 ] echo # Progress should mention at least one target of a computed root grep 'PROG.*BaSeRooT' "${OUT}/log" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/remote.sh000066400000000000000000000107311516554100600273450ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" REMOTE_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" fi echo Using ${REMOTE_ARGS} readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["sh bin/seqfile.sh %d" % (i,)], "deps": ["bin/seqfile.sh"]} targets[""] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "root"} , "root": { "type": "install" , "files": {"TARGETS": "targets", "bin/seqfile.sh": "seqfile"} } , "targets": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS"] , "deps": ["generate.py"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } ] } , "seqfile": {"type": "file_gen", "name": "bin/seqfile.sh", "data": "seq 0 $1 > $1.txt"} } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo echo echo Building computed echo "${JUST}" install ${REMOTE_ARGS} \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 -f "${OUT}/computed.log" \ --main derived -o "${OUT}/derived" 2>&1 echo [ "$(cat "${OUT}/derived/out" | wc -l)" -eq 55 ] # Given the previous build, some form of cache hit is expected grep 'from cache' "${OUT}/computed.log" echo echo Building a different computed root, without previous build echo "${JUST}" install ${REMOTE_ARGS} \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 -f "${OUT}/log" \ --main 'other derived' -o "${OUT}/other-derived" 2>&1 echo [ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ] echo echo Sanity-check of the log echo # As this target was never built before, there can't be a cache hit, # hence we expect a reference to an evaluation log. grep '[Rr]oot.*base.*evaluated.*' "${OUT}/log" > "${TMPDIR}/log_line" cat "${TMPDIR}/log_line" sed -i 's/.*log //' "${TMPDIR}/log_line" "${JUST}" install-cas --local-build-root "${LBRDIR}" \ ${REMOTE_ARGS} \ -o "${OUT}/log.root" $(cat "${TMPDIR}/log_line") echo cat "${OUT}/log.root" echo grep '[Dd]iscovered.*1 action' "${OUT}/log.root" grep '0 cache hit' "${OUT}/log.root" echo echo Building computed root again, expecting root-level cache hit echo "${JUST}" build ${REMOTE_ARGS} \ --local-build-root "${LBRDIR}" -C repo-config.json \ --log-limit 4 -f "${OUT}/log2" \ --main 'other derived' 2>&1 echo grep '[Rr]oot.*from cache' "${OUT}/log2" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/sharding.sh000066400000000000000000000077711516554100600276630ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="${TMPDIR}/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" readonly LOCAL_BIN="${TMPDIR}/local-bin" mkdir -p "${LOCAL_BIN}" export PATH="${LOCAL_BIN}:${PATH}" REMOTE_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" fi echo Using ${REMOTE_ARGS} # To see what was built where, implement a command generateRoot differently # locally and remotely. cat > "${REMOTE_BIN}/generateRoot" <<'EOF' #!/bin/sh cat <<'EOI' { "": { "type": "generic" , "outs": ["out"] , "cmds": ["echo I was defined remotely > out"] } } EOI EOF chmod 755 "${REMOTE_BIN}/generateRoot" cat > "${LOCAL_BIN}/generateRoot" <<'EOF' #!/bin/sh cat <<'EOI' { "": { "type": "generic" , "outs": ["out"] , "cmds": ["echo I was defined locally > out"] } } EOI EOF chmod 755 "${LOCAL_BIN}/generateRoot" # The base root simply refers to the command generateRoot. readonly BASE_ROOT="${ROOT}/base" mkdir -p "${BASE_ROOT}" cd "${BASE_ROOT}" cat > TARGETS <<'EOF' { "": {"type": "export", "target": "root"} , "root": {"type": "generic", "outs": ["TARGETS"], "cmds": ["generateRoot > TARGETS"]} } EOF git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Initial commit" 2>&1 GIT_TREE=$(git log -n 1 --format="%T") mkdir -p "${ROOT}/main" cd "${ROOT}/main" cat > repo-config.json <&1 echo # Now at least the action should be in cache, so the local # tool should not be used any more rm "${LOCAL_BIN}/generateRoot" echo echo Verifying local build echo "${JUST}" install -o "${OUT}/derived-local" \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main derived --log-limit 4 -f "${OUT}/derived-local.log" 2>&1 echo grep 'locally' "${OUT}/derived-local/out" echo echo Build locally again, to be sure the computed root is in cache echo "${JUST}" install -o "${OUT}/derived-local2" \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main derived --log-limit 4 -f "${OUT}/derived-local2.log" 2>&1 echo grep 'locally' "${OUT}/derived-local2/out" echo echo Verify remote build echo "${JUST}" install ${REMOTE_ARGS} -o "${OUT}/derived-remote" \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main derived --log-limit 4 -f "${OUT}/derived-remote.log" 2>&1 echo grep 'remotely' "${OUT}/derived-remote/out" echo # Now, the remote entry is in cache, hence the remote tool should # no longer be used. rm "${REMOTE_BIN}/generateRoot" echo echo Verify remote build, again, ensuring the correct cache is used echo "${JUST}" install ${REMOTE_ARGS} -o "${OUT}/derived-remote2" \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main derived --log-limit 4 -f "${OUT}/derived-remote2.log" 2>&1 echo grep 'remotely' "${OUT}/derived-remote2/out" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/targets.absent-base000066400000000000000000000007031516554100600312730ustar00rootroot00000000000000{ "": {"type": "export", "target": "data", "flexible_config": ["COUNT"]} , "data": { "type": "generic" , "arguments_config": ["COUNT"] , "out_dirs": ["data"] , "cmds": [ { "type": "join" , "$1": [ "COUNT=" , {"type": "join_cmd", "$1": [{"type": "var", "name": "COUNT"}]} ] } , "mkdir -p data" , "for i in `seq 1 $COUNT`" , "do" , " seq 1 $i > data/count$i.txt" , "done" ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/targets.absent-targets000066400000000000000000000002361516554100600320330ustar00rootroot00000000000000{ "cat": { "type": "generic" , "outs": ["out"] , "cmds": ["cat *.txt > out"] , "deps": [["data", ""]] } , "": {"type": "export", "target": "cat"} } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/computed-roots/targets.absent-targets-data000066400000000000000000000000751516554100600327430ustar00rootroot00000000000000{"": {"type": "install", "deps": [["GLOB", null, "*.txt"]]}} just-buildsystem-justbuild-b1fb5fa/test/end-to-end/execution-service/000077500000000000000000000000001516554100600261715ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/execution-service/TARGETS000066400000000000000000000007301516554100600272250ustar00rootroot00000000000000{ "simple_build": { "type": ["@", "rules", "shell/test", "script"] , "name": ["simple_build"] , "test": ["simple_build.sh"] , "deps": [["", "tool-under-test"]] } , "tree_inputs": { "type": ["end-to-end", "with remote"] , "name": ["tree_inputs"] , "test": ["tree_inputs.sh"] , "deps": [["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["execution-service"] , "deps": ["simple_build", "tree_inputs"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/execution-service/simple_build.sh000077500000000000000000000053261516554100600312060ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBRDIR="${PWD}/local-build-root" readonly ESDIR="${PWD}/service-build-root" readonly INFOFILE="${PWD}/info.json" readonly LOGFILE="${PWD}/remote.log" readonly PIDFILE="${PWD}/pid.txt" readonly REFERENCE_OUTPUT="FooOOoo" readonly SERVER_BIN_DIR="${TMPDIR}/server/bin" mkdir -p "${SERVER_BIN_DIR}" cat > "${SERVER_BIN_DIR}/server-only-tool-foo" <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") touch ROOT cat < TARGETS { "": { "type": "generic" , "cmds": ["server-only-tool-foo > foo.txt"] , "outs": ["foo.txt"] } } EOF "${JUST}" install -r localhost:${PORT} --local-build-root="${LBRDIR}" \ --local-launcher '["env", "--", "PATH=/nonexistent"]' \ --remote-instance-name="MyRBEInstance" \ -o . 2>&1 kill $(cat "${PIDFILE}") readonly OUT=$(cat foo.txt) echo "Found output: ${OUT}" if ! [ "${OUT}" = "${REFERENCE_OUTPUT}" ] then printf 'expecting "%s", got "%s"\n' "${REF}" "${OUT}" > /dev/stderr && exit 1 fi echo # Verify logging and that the client sets the instance name correctly grep 'DEBUG' "${LOGFILE}" | grep 'instance_name="MyRBEInstance' > debug-instance.log ## Actions a correct client cannot avoid grep FindMissingBlobs debug-instance.log grep GetActionResult debug-instance.log grep Execute debug-instance.log ## We do not expect the old default or the empty string used as instance name grep 'instance_name=""' "${LOGFILE}" && exit 1 || : grep 'instance_name="remote-execution"' "${LOGFILE}" && exit 1 || : just-buildsystem-justbuild-b1fb5fa/test/end-to-end/execution-service/tree_inputs.sh000077500000000000000000000030731516554100600310740ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBRDIR="${PWD}/local-build-root" touch ROOT # Check if we can collect and stage empty trees cat > TARGETS <<'EOF' { "make_trees": { "type": "generic" , "cmds": ["mkdir -p foo bar/baz"] , "out_dirs": ["."] } , "read_trees": { "type": "generic" , "deps": ["make_trees"] , "cmds": ["set -e", "ls -l foo", "ls -l bar/baz", "echo SUCCESS > result"] , "outs": ["result"] } } EOF REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS}" fi if [ -n "${COMPATIBLE:-}" ]; then REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} --compatible" fi "${JUST}" install ${REMOTE_EXECUTION_ARGS} --local-build-root="${LBRDIR}" -o . read_trees 2>&1 grep SUCCESS result echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/000077500000000000000000000000001516554100600231215ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/TARGETS000066400000000000000000000025151516554100600241600ustar00rootroot00000000000000{ "basic": { "type": ["@", "rules", "shell/test", "script"] , "name": ["basic"] , "test": ["basic.sh"] , "deps": [["", "tool-under-test"]] } , "compactification": { "type": ["@", "rules", "shell/test", "script"] , "name": ["compactification"] , "test": ["compactification.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "export": { "type": ["@", "rules", "shell/test", "script"] , "name": ["export"] , "test": ["export.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "tc-deps": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tc-deps"] , "test": ["tc-deps.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "reconstruct-executable": { "type": ["@", "rules", "shell/test", "script"] , "name": ["reconstruct-executable"] , "test": ["reconstruct-executable.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "gc-all": { "type": ["@", "rules", "shell/test", "script"] , "name": ["gc-all"] , "test": ["gc-all.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["gc"] , "deps": [ "basic" , "compactification" , "export" , "gc-all" , "reconstruct-executable" , "tc-deps" ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/basic.sh000066400000000000000000000102751516554100600245430ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly OUT="${TEST_TMPDIR}/out" BUILD_ARGS="--local-build-root ${LBR}" if [ -n "${COMPATIBLE:-}" ]; then BUILD_ARGS="$BUILD_ARGS --compatible" fi # Have tools in the "outside environment" to later demonstrate # that the action was _not_ rerun. mkdir -p "${TOOLS_DIR}" cat > "${TOOLS_DIR}/tree" <<'EOF' #!/bin/sh mkdir -p $1/hello/world/tree echo Hello World > $1/hello/world/tree/hello.txt echo -n World > $1/hello/world/tree/name.txt EOF chmod 755 "${TOOLS_DIR}/tree" cat > "${TOOLS_DIR}/large" <<'EOF' #!/bin/sh for _ in `seq 1 10000` do echo This is a really large file 1234567890 >> $1 done EOF chmod 755 "${TOOLS_DIR}/large" touch WORKSPACE cat > TARGETS <<'EOF' { "large": { "type": "generic" , "arguments_config": ["ENV"] , "outs": ["out.txt"] , "cmds": ["${TOOLS}/large out.txt"] , "env": {"type": "var", "name": "ENV"} } , "tree": { "type": "generic" , "arguments_config": ["ENV"] , "out_dirs": ["out"] , "cmds": ["${TOOLS}/tree out"] , "env": {"type": "var", "name": "ENV"} } , "": {"type": "install", "dirs": [["large", "out"], ["tree", "out"]]} } EOF cat TARGETS # Build to fill the cache "${JUST}" build ${BUILD_ARGS} -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # Demonstrate that from now on, we don't build anything any more rm -rf "${TOOLS_DIR}" # Verify the large file is in cache "${JUST}" install ${BUILD_ARGS} -L '["env", "PATH='"${PATH}"'"]' -o "${OUT}/out-large" \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' large 2>&1 wc -c "${OUT}/out-large/out.txt" test $(cat "${OUT}/out-large/out.txt" | wc -c) -gt 100000 # Now test non-rotating gc; this does not affect how far the # cache. At the end, we also repeat it serveral time to demontstrate # that things don't fall out of cache. LEFT_OVER_REMOVE_DIR="${LBR}/protocol-dependent/remove-me-$$-xxx" EPHEMERAL_DIR="${LBR}/protocol-dependent/generation-0/ephemeral" mkdir -p "${LEFT_OVER_REMOVE_DIR}/xxx" mkdir -p "${EPHEMERAL_DIR}/xxx" "${JUST}" gc --local-build-root "${LBR}" --no-rotate 2>&1 echo "Checking that ${EPHEMERAL_DIR} was removed" [ -e "${EPHEMERAL_DIR}" ] && exit 1 || : echo "Checking that ${LEFT_OVER_REMOVE_DIR} was removed" [ -e "${LEFT_OVER_REMOVE_DIR}" ] && exit 1 || : "${JUST}" gc --local-build-root "${LBR}" --no-rotate 2>&1 "${JUST}" gc --local-build-root "${LBR}" --no-rotate 2>&1 # collect garbage "${JUST}" gc --local-build-root "${LBR}" 2>&1 # Use the tree "${JUST}" build ${BUILD_ARGS} -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' tree 2>&1 # collect garbage again "${JUST}" gc --local-build-root "${LBR}" 2>&1 # Verify the build root is now small tar cvf "${OUT}/root.tar" "${LBR}" 2>&1 wc -c "${OUT}/root.tar" test $(cat "${OUT}/root.tar" | wc -c) -lt 100000 # Verify that the tree is fully in cache "${JUST}" install ${BUILD_ARGS} -L '["env", "PATH='"${PATH}"'"]' -o "${OUT}/out-tree" \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' tree 2>&1 ls -R "${OUT}/out-tree" test -f "${OUT}/out-tree/out/hello/world/tree/hello.txt" test -f "${OUT}/out-tree/out/hello/world/tree/name.txt" test "$(cat "${OUT}/out-tree/out/hello/world/tree/name.txt")" = "World" # check if all files in generation 0 have been linked and not copied readonly NON_LINKED_FILES=$(find ${LBR}/protocol-dependent/generation-0 -type f -exec sh -c 'test $(stat -c %h {}) != 2' \; -print) echo echo "Files with link count!=2:" echo "${NON_LINKED_FILES:-""}" echo test -z "${NON_LINKED_FILES}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/compactification.sh000066400000000000000000000120351516554100600267720ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly JUST_ARGS="--local-build-root ${LBR}" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR_MR="${TEST_TMPDIR}/local-build-root-mr" readonly JUST_MR_ARGS="--norc --local-build-root ${LBR_MR} --just ${JUST}" STORAGE="${LBR}/protocol-dependent/generation-0" COMPATIBLE_ARGS="" if [ -n "${COMPATIBLE:-}" ]; then COMPATIBLE_ARGS="--compatible" STORAGE="${STORAGE}/compatible-sha256" else STORAGE="${STORAGE}/git-sha1" fi readonly TOOLS_DIR="${TEST_TMPDIR}/tools" mkdir -p "${TOOLS_DIR}" cat > "${TOOLS_DIR}/make_large.sh" <<'EOF' #!/bin/sh readonly Mb=$((1024 * 1024)) readonly COUNT=$((${1} * ${Mb} / 8)) for _ in `seq 1 ${COUNT}` do echo 01234567 done EOF chmod 755 "${TOOLS_DIR}/make_large.sh" mkdir work && cd work touch ROOT cat > repos.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat > TARGETS <<'EOF' { "large": { "type": "generic" , "arguments_config": ["ENV"] , "outs": ["out.txt"] , "cmds": ["${TOOLS}/make_large.sh 5 > out.txt"] , "env": {"type": "var", "name": "ENV"} } , "large_export": { "type": "export" , "flexible_config": ["ENV"] , "target": "large" } , "main": { "type": "generic" , "arguments_config": ["ENV"] , "outs": ["dummy.txt"] , "cmds": ["cat out.txt out.txt > dummy.txt"] , "env": {"type": "var", "name": "ENV"} , "deps": ["large_export"] } , "": {"type": "install", "dirs": [["large", "out"], ["main", "out"]]} } EOF cat TARGETS readonly GRPC_THRESHOLD_MB=4 get_large_files_count() { find "${STORAGE}" -size +${GRPC_THRESHOLD_MB}M | wc -l } get_lbr_storage_size() { du -hs --block-size=1 "${STORAGE}" | cut -f 1 } print_storage_statistics() { local size="$(get_lbr_storage_size)" local count="$(get_large_files_count)" echo "SIZE STORAGE: ${size}" echo "COUNT OF FILES LARGER THAN ${GRPC_THRESHOLD_MB} MB IS: ${count}" } # Build to fill the cache "${JUST_MR}" ${JUST_MR_ARGS} build ${JUST_ARGS} ${COMPATIBLE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # Demonstrate that from now on, we don't build anything any more rm -rf "${TOOLS_DIR}" readonly SIZE_BEFORE="$(get_lbr_storage_size)" readonly COUNT_BEFORE="$(get_large_files_count)" echo 'BEFORE COMPACTIFICATION:' print_storage_statistics [ ${COUNT_BEFORE} -gt 0 ] && [ ${COUNT_BEFORE} -le 3 ] # Run compactification of the last generation. No large files must remain. "${JUST}" gc ${JUST_ARGS} --no-rotate readonly COMPACTIFIED_SIZE=$(get_lbr_storage_size) readonly COUNT_COMPACTIFIED=$(get_large_files_count) echo 'AFTER COMPACTIFICATION:' print_storage_statistics [ ${COMPACTIFIED_SIZE} -le ${SIZE_BEFORE} ] [ ${COUNT_COMPACTIFIED} -eq 0 ] # Build one more time to ensure that for fully cached builds nothing except export targets gets reconstructed "${JUST_MR}" ${JUST_MR_ARGS} build ${JUST_ARGS} ${COMPATIBLE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 readonly RECONSTRUCTED_SIZE=$(get_lbr_storage_size) readonly RECONSTRUCTED_COUNT=$(get_large_files_count) echo 'AFTER RECONSTRUCTION:' print_storage_statistics # Reconstruction of export targets is allowed: [ ${RECONSTRUCTED_COUNT} -le 1 ] readonly EXPORT_PATH=$(find "${STORAGE}" -size +${GRPC_THRESHOLD_MB}M) readonly EXPORT_SIZE=$(du -hs --block-size=1 "${EXPORT_PATH}" | cut -f 1) readonly EXPECTED_SIZE=$((${COMPACTIFIED_SIZE}+${EXPORT_SIZE})) echo "EXPECTED SIZE IS ${EXPECTED_SIZE}" [ ${RECONSTRUCTED_SIZE} -le ${EXPECTED_SIZE} ] # Rotation and building again should not reconstruct anything except export targets. "${JUST}" gc ${JUST_ARGS} "${JUST_MR}" ${JUST_MR_ARGS} build ${JUST_ARGS} ${COMPATIBLE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 readonly ROTATED_BUILD_SIZE=$(get_lbr_storage_size) readonly ROTATED_BUILD_COUNT=$(get_large_files_count) echo 'AFTER ROTATION AND BUILD:' print_storage_statistics [ ${ROTATED_BUILD_SIZE} -le ${EXPECTED_SIZE} ] [ ${ROTATED_BUILD_COUNT} -le 1 ] # Calculate the size difference. # The resulting size after rotation must not exceed the size of compactification + the size of the export target: readonly SIZE_DIFFERENCE=$((${ROTATED_BUILD_SIZE}-${COMPACTIFIED_SIZE}-${EXPORT_SIZE})) echo "SIZE DIFFERENCE IS ${SIZE_DIFFERENCE}" [ ${SIZE_DIFFERENCE} -le 0 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/export.sh000066400000000000000000000064361516554100600250070ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_MR="${TEST_TMPDIR}/local-build-root-mr" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly OUT="${TEST_TMPDIR}/out" readonly JUST_MR_ARGS="--norc --just ${JUST} --local-build-root ${LBR_MR}" BUILD_ARGS="--local-build-root ${LBR}" if [ -n "${COMPATIBLE:-}" ]; then BUILD_ARGS="$BUILD_ARGS --compatible" fi # Have tools in the "outside environment" to later demonstrate # that the action was _not_ rerun. mkdir -p "${TOOLS_DIR}" cat > "${TOOLS_DIR}/tree" <<'EOF' #!/bin/sh mkdir -p $1/hello/world/tree echo Hello World > $1/hello/world/tree/hello.txt echo -n World > $1/hello/world/tree/name.txt EOF chmod 755 "${TOOLS_DIR}/tree" mkdir work cd work touch WORKSPACE cat > TARGETS <<'EOF' { "tree": { "type": "generic" , "arguments_config": ["ENV"] , "out_dirs": ["out"] , "cmds": ["${TOOLS}/tree out"] , "env": {"type": "var", "name": "ENV"} } , "": { "type": "export" , "flexible_config": ["ENV"] , "target": "tree" } } EOF cat > repos.json <<'EOF' { "main": "" , "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat repos.json cat TARGETS # Build to fill the cache "${JUST_MR}" ${JUST_MR_ARGS} build ${BUILD_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # Demonstrate that from now on, we don't build anything any more rm -rf "${TOOLS_DIR}" # Demonstrate that we can build the export target without any cached actions rm -rf ${LBR}/protocol-dependent/generation-*/*/ac # collect garbage "${JUST_MR}" ${JUST_MR_ARGS} gc --local-build-root ${LBR} 2>&1 # Use the export "${JUST_MR}" ${JUST_MR_ARGS} build ${BUILD_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # collect garbage again "${JUST_MR}" ${JUST_MR_ARGS} gc --local-build-root ${LBR} 2>&1 # Verify that the export target is fully in cache "${JUST_MR}" ${JUST_MR_ARGS} install ${BUILD_ARGS} -o "${OUT}" \ -L '["env", "PATH='"${PATH}"'"]' \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 ls -R "${OUT}" test -f "${OUT}/out/hello/world/tree/hello.txt" test -f "${OUT}/out/hello/world/tree/name.txt" test "$(cat "${OUT}/out/hello/world/tree/name.txt")" = "World" # check if all files in generation 0 have been linked and not copied readonly NON_LINKED_FILES=$(find ${LBR}/protocol-dependent/generation-0 -type f -exec sh -c 'test $(stat -c %h {}) != 2' \; -print) echo echo "Files with link count!=2:" echo "${NON_LINKED_FILES:-""}" echo test -z "${NON_LINKED_FILES}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/gc-all.sh000066400000000000000000000043401516554100600246150ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly JUST_ARGS="--local-build-root ${LBR}" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR_MR="${TEST_TMPDIR}/local-build-root-mr" readonly JUST_MR_ARGS="--norc --local-build-root ${LBR_MR} --just ${JUST}" readonly OUT="${TEST_TMPDIR}/out" mkdir -p "${OUT}" STORAGE="${LBR}/protocol-dependent/generation-0" COMPATIBLE_ARGS="" if [ -n "${COMPATIBLE:-}" ]; then COMPATIBLE_ARGS="--compatible" fi mkdir work && cd work touch ROOT cat > repos.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat > TARGETS <<'EOF' { "file": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo foo > out.txt"] } , "": {"type": "install", "dirs": [["file", "out"]]} } EOF cat TARGETS # Build to fill the cache "${JUST_MR}" ${JUST_MR_ARGS} build ${JUST_ARGS} ${COMPATIBLE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' 2>&1 if [ ! -d ${STORAGE} ]; then echo "Storage has not been created" exit 1 fi; # Run regular gc with --no-rotate to ensure compactification gets triggered: "${JUST_MR}" ${JUST_MR_ARGS} gc ${JUST_ARGS} --no-rotate --log-limit 4 \ -f "${OUT}/gc" 2>&1 grep 'Compactification has been started' "${OUT}/gc" # Run gc with --all to ensure compactification doesn't get triggered: "${JUST_MR}" ${JUST_MR_ARGS} gc ${JUST_ARGS} --all --log-limit 4 \ -f "${OUT}/gc2" 2>&1 grep 'Compactification has been started' "${OUT}/gc2" && exit 1 if [ -d ${STORAGE} ]; then echo "Storage has not been deleted" exit 1 fi; echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/reconstruct-executable.sh000066400000000000000000000102151516554100600301460ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly JUST_MR_ARGS="--norc --local-build-root ${LBR} --just ${JUST}" mkdir repo cd repo touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > generate-data.py <<'EOF' #!/usr/bin/env python3 # Generates a C source file where we know the binary # will be larger than the compactification threshold, as # it contains enough data embedded. import random import sys count = 4 * 1024 * 1024 random.seed(sys.argv[1]) print("#include ") print("int main(int argc, char **argv) {") print(" char data[%d] = {" % (count+1,)) bytes_to_embed = random.randbytes(count); for c in bytes_to_embed: print(" '\\x%02x'," %(c,)) print(" '\\x00'};"); print(" int i;") print(" for (i=0; i < %d; i++) {" % (count,)); print(" printf(\"%c\", data[i]);") print(" }") print(" return 0;") print("}") EOF chmod 755 generate-data.py # The default target depends on many large binaries that need to be run cat > TARGETS <<'EOF' { "source": { "type": "generic" , "arguments_config": ["SEED"] , "outs": ["src.c"] , "deps": ["generate-data.py"] , "cmds": [ { "type": "join" , "$1": [ { "type": "join_cmd" , "$1": ["./generate-data.py", {"type": "var", "name": "SEED"}] } , "> src.c" ] } ] } , "binary": { "type": "generic" , "outs": ["generate"] , "deps": ["source"] , "cmds": ["cc -o generate src.c"] } , "data": { "type": "generic" , "outs": ["data"] , "deps": ["binary"] , "cmds": ["./generate > data"] } , "aaa": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "aaa"} } , "bbb": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "bbb"} } , "ccc": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "ccc"} } , "ddd": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "ddd"} } , "eee": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "eee"} } , "fff": { "type": "configure" , "target": "data" , "config": {"type": "singleton_map", "key": "SEED", "value": "fff"} } , "": { "type": "install" , "files": { "aaa": "aaa" , "bbb": "bbb" , "ccc": "ccc" , "ddd": "ddd" , "eee": "eee" , "fff": "fff" } } } EOF # First we build all the relevant binaries to have them in test "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "aaa"}' binary 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "bbb"}' binary 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "ccc"}' binary 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "ddd"}' binary 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "eee"}' binary 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -D '{"SEED": "fff"}' binary 2>&1 # Now, compactify "${JUST_MR}" ${JUST_MR_ARGS} gc --no-rotate # Finally, run the default target; this will, in parallel, reconstruct the # compactified binaries and use them "${JUST_MR}" ${JUST_MR_ARGS} build -L '["env", "PATH='"${PATH}"'"]' -J20 2>&1 echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/gc/tc-deps.sh000066400000000000000000000072541516554100600250240ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_MR="${TEST_TMPDIR}/local-build-root-mr" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly OUT="${TEST_TMPDIR}/out" readonly JUST_MR_ARGS="--norc --just ${JUST} --local-build-root ${LBR_MR}" JUST_ARGS="--local-build-root ${LBR}" BUILD_ARGS="${JUST_ARGS} --log-limit 4" if [ -n "${COMPATIBLE:-}" ]; then BUILD_ARGS="$BUILD_ARGS --compatible" fi mkdir work cd work touch ROOT # general setup: 2 export targets ["@", "foo", "", ""] and ["@", "bar", "", ""] # with a common dependency, the export target ["@", "common", "", ""] cat > repos.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "top-level", "pragma": {"to_git": true}} , "bindings": {"foo": "foo", "bar": "bar", "common": "common"} } , "foo": { "repository": {"type": "file", "path": "foo", "pragma": {"to_git": true}} , "bindings": {"common": "common"} } , "bar": { "repository": {"type": "file", "path": "bar", "pragma": {"to_git": true}} , "bindings": {"common": "common"} } , "common": { "repository": {"type": "file", "path": "common", "pragma": {"to_git": true}} } } } EOF mkdir common cat > common/TARGETS <<'EOF' { "common.txt": { "type": "generic" , "outs": ["common.txt"] , "cmds": ["echo common > common.txt"] } , "common": {"type": "install", "files": {"common.txt": "common.txt"}} , "": {"type": "export", "target": "common"} } EOF mkdir foo cat > foo/TARGETS <<'EOF' { "foo.txt": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo foo > foo.txt"]} , "foo": { "type": "install" , "files": {"foo.txt": "foo.txt", "common.txt": ["@", "common", "", ""]} } , "": {"type": "export", "target": "foo"} } EOF mkdir bar cat > bar/TARGETS <<'EOF' { "bar.txt": {"type": "generic", "outs": ["bar.txt"], "cmds": ["echo bar > bar.txt"]} , "bar": { "type": "install" , "files": {"bar.txt": "bar.txt", "common.txt": ["@", "common", "", ""]} } , "": {"type": "export", "target": "bar"} } EOF mkdir top-level cat > top-level/TARGETS <<'EOF' {"": { "type": "install" , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF # Test the interaction with gc echo echo 'First build, gets foo, bar, common into tc' echo "${JUST_MR}" ${JUST_MR_ARGS} build ${BUILD_ARGS} 2>&1 echo echo gc to put all into the old generation echo "${JUST}" gc ${JUST_ARGS} 2>&1 echo echo 'build again; this gets foo and bar in the young generation' echo "${JUST_MR}" ${JUST_MR_ARGS} build ${BUILD_ARGS} 2>&1 echo echo 'gc again; this would (without invariants) get common out' echo "${JUST}" gc ${JUST_ARGS} 2>&1 echo echo 'Modify the root of bar' echo touch bar/bar_root_has_changed echo echo 'Analyse the relevant targets' echo "${JUST_MR}" ${JUST_MR_ARGS} --main foo analyse ${BUILD_ARGS} 2>&1 "${JUST_MR}" ${JUST_MR_ARGS} --main bar analyse ${BUILD_ARGS} 2>&1 echo echo 'build again; this checks for staging conflics' echo "${JUST_MR}" ${JUST_MR_ARGS} build ${BUILD_ARGS} 2>&1 echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/000077500000000000000000000000001516554100600257505ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/TARGETS000066400000000000000000000011521516554100600270030ustar00rootroot00000000000000{ "shell": { "type": ["@", "rules", "shell/test", "script"] , "name": ["shell"] , "test": ["shell-script.sh"] , "keep": ["out/out.txt"] , "deps": [["", "tool-under-test"]] } , "compile rules": {"type": "install", "files": {"RULES": "data/RULES.compiled"}} , "compiled": { "type": ["@", "rules", "shell/test", "script"] , "name": ["compiled"] , "test": ["compiled.sh"] , "keep": ["graph.json", "out/out.txt"] , "deps": ["compile rules", ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["generated-binary"] , "deps": ["compiled", "shell"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/compiled.sh000077500000000000000000000021551516554100600301060ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir .tool-root touch ROOT cat > TARGETS <<'EOI' { "program outputs": {"type": "generated files", "width": ["16"]} , "ALL": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["cat $(ls out-*.txt | sort) > out.txt"] , "deps": ["program outputs"] } } EOI echo echo "Analysing" bin/tool-under-test analyse --local-build-root .tool-root --dump-graph graph.json 2>&1 echo echo "Building" bin/tool-under-test install -L '["env", "PATH='"${PATH}"'"]' -o out --local-build-root .tool-root -J 16 2>&1 just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/data/000077500000000000000000000000001516554100600266615ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/data/RULES.compiled000066400000000000000000000042731516554100600312770ustar00rootroot00000000000000{ "generated files": { "string_fields": ["width"] , "expression": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": { "type": "foreach" , "var": "n" , "range": { "type": "range" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "width"}} } , "body": { "type": "let*" , "bindings": [ [ "src" , { "type": "join" , "$1": [ "#include \n" , "int main(int argc, char **argv) {\n" , " printf(\"Hello from generated program #%s!\", \"" , {"type": "var", "name": "n"} , "\");\n" , " return 0;\n" , "}\n" ] } ] , [ "src file" , { "type": "singleton_map" , "key": "hello.c" , "value": {"type": "BLOB", "data": {"type": "var", "name": "src"}} } ] , [ "hello" , { "type": "ACTION" , "inputs": {"type": "var", "name": "src file"} , "outs": ["hello"] , "cmd": ["cc", "-o", "hello", "hello.c"] , "env": { "type": "singleton_map" , "key": "PATH" , "value": "/bin:/sbin:/usr/bin:/usr/sbin" } } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "hello"} , "outs": [ { "type": "join" , "$1": ["out-", {"type": "var", "name": "n"}, ".txt"] } ] , "cmd": [ "/bin/sh" , "-c" , { "type": "join" , "$1": ["./hello > out-", {"type": "var", "name": "n"}, ".txt"] } ] } ] ] , "body": {"type": "var", "name": "out"} } } } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/generated-binary/shell-script.sh000077500000000000000000000032321516554100600307200ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir .tool-root touch ROOT cat > TARGETS <<'EOI' { "script-hello": { "type": "generic" , "outs": ["script.sh"] , "cmds": [ "echo '#!/bin/sh' > script.sh" , "echo 'echo Hello World' >> script.sh" , "chmod 755 script.sh" ] } , "generated-hello": { "type": "generic" , "outs": ["out-hello.txt"] , "deps": ["script-hello"] , "cmds": ["./script.sh > out-hello.txt"] } , "script-morning": { "type": "generic" , "outs": ["script.sh"] , "cmds": [ "echo '#!/bin/sh' > script.sh" , "echo 'echo Good morning' >> script.sh" , "chmod 755 script.sh" ] } , "generated-morning": { "type": "generic" , "outs": ["out-morning.txt"] , "deps": ["script-morning"] , "cmds": ["./script.sh > out-morning.txt"] } , "ALL": {"type": "generic" , "deps": ["generated-hello", "generated-morning"] , "outs": ["out.txt"] , "cmds": ["cat out-*.txt > out.txt"] } } EOI bin/tool-under-test install -L '["env", "PATH='"${PATH}"'"]' -o out --local-build-root .tool-root 2>&1 grep Hello out/out.txt grep Good out/out.txt just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/000077500000000000000000000000001516554100600246235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/TARGETS000066400000000000000000000052321516554100600256610ustar00rootroot00000000000000{ "chained-import": { "type": ["@", "rules", "shell/test", "script"] , "name": ["chained-import"] , "test": ["chained-import.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "git-import-under-test"] ] } , "deduplicate": { "type": ["@", "rules", "shell/test", "script"] , "name": ["deduplicate"] , "test": ["deduplicate.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "deduplicate-tool-under-test"] , ["end-to-end", "git-import-under-test"] ] , "keep": ["repos-full.json", "actions-full.json", "repos.json", "actions.json"] } , "annotations": { "type": ["@", "rules", "shell/test", "script"] , "name": ["annotations"] , "test": ["annotations.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "git-import-under-test"] ] , "keep": ["repos.json"] } , "absent": { "type": ["@", "rules", "shell/test", "script"] , "name": ["absent"] , "test": ["absent.sh"] , "deps": [ ["end-to-end", "deduplicate-tool-under-test"] , ["end-to-end", "git-import-under-test"] ] , "keep": ["repos.json", "deduplicated.json"] } , "computed": { "type": ["@", "rules", "shell/test", "script"] , "name": ["computed"] , "test": ["computed.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "deduplicate-tool-under-test"] , ["end-to-end", "git-import-under-test"] ] , "keep": ["repos-full.json", "actions-full.json", "repos.json", "actions.json"] } , "tree_structure": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tree_structure"] , "test": ["tree_structure.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "deduplicate-tool-under-test"] , ["end-to-end", "git-import-under-test"] ] } , "indirections": { "type": ["@", "rules", "shell/test", "script"] , "name": ["indirections"] , "test": ["indirections.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "git-import-under-test"] ] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "stage": ["git-import"] , "deps": { "type": "++" , "$1": [ ["deduplicate", "absent"] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": [ "chained-import" , "annotations" , "computed" , "indirections" , "tree_structure" ] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/absent.sh000066400000000000000000000075571516554100600264510ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly DEDUPLICATE="${PWD}/bin/deduplicate-tool-under-test" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo/" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": "src"}, "bindings": {"bar": "bar"}} , "bar": { "repository": { "type": "archive" , "content": "4d86bdf4d07535db3754e6cbf30a6fc81aaa2cc1" , "fetch": "https://example.org/v1.2.3" } } } } EOF git init 2>&1 git checkout --orphan foomaster 2>&1 git config user.name 'N.O.Body' 2>&1 git config user.email 'nobody@example.org' 2>&1 git add . 2>&1 git commit -m 'Add foo' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": { "foo": "foo" , "bar": "my/internal/bar/version" , "file-foo": "file-foo" , "file-bar": "file-bar" } } , "my/internal/bar/version": { "repository": { "type": "archive" , "content": "4d86bdf4d07535db3754e6cbf30a6fc81aaa2cc1" , "fetch": "https://example.org/v1.2.3" } } , "file-foo": { "repository": {"type": "file", "path": "third_party", "pragma": {"to_git": true}} } , "file-bar": {"repository": {"type": "file", "path": "third_party"}} } } EOF echo echo Import echo "${GIT_IMPORT}" -C repos.template.json \ --absent \ --as foo -b foomaster \ --mirror http://primary.example.org/foo \ --inherit-env AUTH_VAR_FOO \ "${REPO_DIRS}/foo" \ > repos.json echo cat repos.json echo # Take repository foo jq '.repositories.foo' repos.json > foo.json # - repository should be absent [ $(jq '.repository.pragma.absent' foo.json) = "true" ] # - get repository name of binding "bar" BAR=$(jq '.bindings.bar' foo.json) echo "Name of binding bar of foo: $BAR" # - ... this repo should be absent as well jq '.repositories.'"$BAR" repos.json > bar.json cat bar.json [ $(jq '.repository.pragma.absent' bar.json) = "true" ] echo echo Deduplicate echo cat repos.json | "${DEDUPLICATE}" > deduplicated.json echo cat deduplicated.json echo # Both bar repos should be unified now FOO_BAR=$(jq '.repositories.foo.bindings.bar' deduplicated.json) echo "Bar repo of foo: ${FOO_BAR}" INTERNAL_BAR=$(jq '.repositories."".bindings.bar' deduplicated.json) echo "Internal bar repo: ${INTERNAL_BAR}" [ "${FOO_BAR}" = "${INTERNAL_BAR}" ] jq '.repositories.'"$FOO_BAR" deduplicated.json > bar.json cat bar.json # ... and the pargma empty, as the internal bar is part of the merge # and that repository cannot be absent. [ $(jq '.pragma?' bar.json) = "null" ] # Also, both file repositories should be unified FILE_FOO=$(jq '.repositories."".bindings."file-foo"' deduplicated.json) echo "file-foo of '' is now ${FILE_FOO}" FILE_BAR=$(jq '.repositories."".bindings."file-bar"' deduplicated.json) echo "file-bar of '' is now ${FILE_BAR}" [ "${FILE_FOO}" = "${FILE_BAR}" ] jq '.repositories.'"$FILE_FOO" deduplicated.json > local.json cat local.json # The to_git pragma should be preserved [ $(jq '.repository.pragma."to_git"' local.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/annotations.sh000077500000000000000000000053701516554100600275240ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly OUT="${TEST_TMPDIR}/out" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "src"}}}} EOF cat > src/TARGETS <<'EOF' { "": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo -n FOO > foo.txt"]} } EOF git init 2>&1 git checkout --orphan foomaster 2>&1 git config user.name 'N.O.Body' 2>&1 git config user.email 'nobody@example.org' 2>&1 git add . 2>&1 git commit -m 'Add foo.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["bar.txt"] , "cmds": ["cat foo.txt | tr A-Z a-z > bar.txt"] , "deps": [["@", "foo", "", ""]] } } EOF cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo"} } } } EOF echo echo Import echo "${GIT_IMPORT}" -C repos.template.json \ --as foo -b foomaster \ --mirror http://primary.example.org/foo \ --inherit-env AUTH_VAR_FOO \ --mirror http://secondary.example.org/foo \ --inherit-env AUTH_VAR_BAR \ "${REPO_DIRS}/foo" \ > repos.json echo cat repos.json echo echo echo Sanity check: can build echo "${JUST_MR}" --norc --just "${JUST}" \ -L '["env", "PATH='"${PATH}"'"]' \ -C repos.json \ --local-build-root "${LBR}" \ install -o "${OUT}" 2>&1 [ "$(cat ${OUT}/bar.txt)" = "foo" ] echo echo Build OK, verifying annotations echo jq '.repositories.foo.repository' repos.json > foo.json echo Repository entry for foo echo cat foo.json echo [ $(jq '."inherit env" | . == ["AUTH_VAR_FOO", "AUTH_VAR_BAR"]' foo.json) = "true" ] [ $(jq '."mirrors" | . == ["http://primary.example.org/foo", "http://secondary.example.org/foo"]' foo.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/chained-import.sh000077500000000000000000000047471516554100600301010ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "src"}}}} EOF cat > src/TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": ""}}}} EOF cat > TARGETS <<'EOF' { "": {"type": "file_gen", "name": "bar.txt", "data": "BAR"}} EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo", "bar": "bar"} } } } EOF "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" > repos.json echo cat repos.json echo "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q FOOBAR "${OUT}/out.txt" echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/computed.sh000066400000000000000000000137561516554100600270130ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly DEDUPLICATE="${PWD}/bin/deduplicate-tool-under-test" readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" # Set up repo foo mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"to_git": true} } } } } EOF cat > src/TARGETS <<'EOF' { "": {"type": "export", "target": "gen"} , "gen": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo -n FOO > foo.txt"]} } EOF git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 # Set up repo bar mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.template.json <<'EOF' { "repositories": { "": {"repository": "inner", "target_root": "computed_src"} , "inner": { "repository": {"type": "file", "path": ".", "pragma": {"to_git": true}} , "bindings": {"DoNotImport": "Missing"} } , "src": { "repository": {"type": "file", "path": "src", "pragma": {"to_git": true}} , "bindings": {"foo": "foo"} } , "computed_src": { "repository": { "type": "computed" , "repo": "src" , "target": ["", ""] , "config": {"COUNT": "10"} } } , "root_inner": { "repository": {"type": "file", "path": ".", "pragma": {"to_git": true}} , "bindings": {"AlsoDoNotImport": "AlsoMissing"} } , "root": {"repository": "root_inner", "bindings": {"foo": "foo"}} , "bar_root": { "repository": { "type": "computed" , "repo": "root" , "target": ["", ""] , "config": {"COUNT": "12"} } } } } EOF "${GIT_IMPORT}" -C repos.template.json \ --as foo -b foomaster "${REPO_DIRS}/foo" > repos.json cat repos.json cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "export", "target": "gen"} targets["gen"] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS", "bar.txt"] , "deps": ["generate.py", ["@", "foo", "", ""]] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } , "cat foo.txt | tr A-Z a-z > bar.txt" ] } } EOF mkdir src cp generate.py src/generate.py cp TARGETS src/TARGETS git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 # Set up repo to build mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": {"type": "export", "target": "gen"} , "gen": { "type": "generic" , "cmds": ["cat bar.txt init.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "bar_root", "", ""], "init"] } , "init": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > init.txt"] , "outs": ["init.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": ".", "pragma": {"to_git": true}} , "bindings": {"foo": "foo", "bar": "bar", "bar_root": "bar_root"} } } } EOF "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" \ | "${GIT_IMPORT}" -C - -b barmaster "${REPO_DIRS}/bar" bar_root \ > repos-full.template.json # Test deduplication takes into account target repos of precomputed roots # as well. 'repo' isn't used directly, but it can shadow 'bar_root/root' # during deduplication: "${GIT_IMPORT}" -C repos-full.template.json -b barmaster "${REPO_DIRS}/bar" \ root > repos-full.json echo cat repos-full.json grep DoNotImport && exit 1 || : # we should not bring in unneeded binding echo "${JUST_MR}" -C repos-full.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ -L '["env", "PATH='"${PATH}"'"]' \ --dump-plain-graph actions-full.json 2>&1 echo cat repos-full.json | "${DEDUPLICATE}" > repos.json cat repos.json echo "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ -L '["env", "PATH='"${PATH}"'"]' \ --dump-plain-graph actions.json 2>&1 # Verify that we reduced the number of repositories, but did # not change the action graph (except for the origins of the actions). [ $(jq -aM '.repositories | length' repos.json) -lt $(jq -aM '.repositories | length' repos-full.json) ] cmp actions-full.json actions.json echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/deduplicate.sh000077500000000000000000000064361516554100600274560ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly DEDUPLICATE="${PWD}/bin/deduplicate-tool-under-test" readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "src"}}}} EOF cat > src/TARGETS <<'EOF' { "": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo -n FOO > foo.txt"]} } EOF git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.template.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": ""}, "bindings": {"foo": "foo"}} } } EOF "${GIT_IMPORT}" -C repos.template.json \ --as foo -b foomaster "${REPO_DIRS}/foo" > repos.json cat repos.json cat > TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["bar.txt"] , "cmds": ["cat foo.txt | tr A-Z a-z > bar.txt"] , "deps": [["@", "foo", "", ""]] } } EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo", "bar": "bar"} } } } EOF "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" \ > repos-full.json echo cat repos-full.json echo "${JUST_MR}" -C repos-full.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ --dump-plain-graph actions-full.json 2>&1 echo cat repos-full.json | "${DEDUPLICATE}" > repos.json cat repos.json echo "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ --dump-plain-graph actions.json 2>&1 # Verify that we reduced the number of repositories, but did # not change the action graph (except for the origins of the actions). [ $(jq -aM '.repositories | length' repos.json) -lt $(jq -aM '.repositories | length' repos-full.json) ] cmp actions-full.json actions.json echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/indirections.sh000066400000000000000000000107341516554100600276560ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly OUT="${TEST_TMPDIR}/out" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "main"} , "bindings": {"rules": "rules"} } , "rules orig": {"repository": {"type": "file", "path": "rules"}} , "defaults": {"repository": {"type": "file", "path": "defaults"}} , "rules": { "repository": "rules orig" , "target_root": "defaults" , "rule_root": "rules" } } } EOF mkdir main cat > main/TARGETS <<'EOF' {"": {"type": ["@", "rules", "", "normalize"], "file": ["foo.txt"]}} EOF echo foo > main/foo.txt mkdir rules cat > rules/RULES <<'EOF' { "defaults": { "string_fields": ["cmd"] , "expression": { "type": "RESULT" , "provides": { "type": "singleton_map" , "key": "cmd" , "value": {"type": "FIELD", "name": "cmd"} } } } , "normalize": { "target_fields": ["file"] , "implicit": {"defaults": ["defaults"]} , "expression": { "type": "let*" , "bindings": [ [ "in" , { "type": "disjoint_map_union" , "$1": { "type": "++" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "file"} , "body": { "type": "foreach_map" , "range": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "_"} } , "body": { "type": "singleton_map" , "key": "in" , "value": {"type": "var", "name": "$_"} } } } } } ] , [ "cmd" , { "type": "++" , "$1": { "type": "foreach" , "range": {"type": "FIELD", "name": "defaults"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "_"} , "provider": "cmd" } } } ] , [ "sh cmd" , { "type": "join" , "$1": [ "cat in | " , {"type": "join_cmd", "$1": {"type": "var", "name": "cmd"}} , " > out" ] } ] , [ "out" , { "type": "ACTION" , "inputs": {"type": "var", "name": "in"} , "outs": ["out"] , "cmd": ["sh", "-c", {"type": "var", "name": "sh cmd"}] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "out"}} } } } EOF mkdir defaults cat > defaults/TARGETS <<'EOF' {"defaults": {"type": "defaults", "cmd": ["tr", "a-z", "A-Z"]}} EOF git init 2>&1 git checkout --orphan foomaster 2>&1 git config user.name 'N.O.Body' 2>&1 git config user.email 'nobody@example.org' 2>&1 git add . 2>&1 git commit -m 'Add normalized foot' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' {"": {"type": "install", "files": {"normalized/foo": ["@", "foo", "", ""]}}} EOF cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo"} } } } EOF echo echo Import echo "${GIT_IMPORT}" -C repos.template.json \ --as foo -b foomaster \ "${REPO_DIRS}/foo" \ > repos.json echo cat repos.json echo echo echo Check: can build echo "${JUST_MR}" --norc --just "${JUST}" \ -L '["env", "PATH='"${PATH}"'"]' \ -C repos.json \ --local-build-root "${LBR}" \ install -o "${OUT}" 2>&1 grep FOO "${OUT}/normalized/foo" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/git-import/tree_structure.sh000066400000000000000000000130001516554100600302300ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly DEDUPLICATE="${ROOT}/bin/deduplicate-tool-under-test" readonly GIT_IMPORT="${ROOT}/bin/git-import-under-test" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi # Set up repo foo readonly FOO_REPO="${REPO_DIRS}/foo" readonly FOO_NESTED_DIR="${FOO_REPO}/src/nested_dir/nested_dir_2" mkdir -p "${FOO_NESTED_DIR}" echo "content" > "${FOO_NESTED_DIR}/file" cd "${FOO_REPO}" echo echo "Creating TARGETS at ${FOO_REPO}:" cat > TARGETS << 'EOF' { "": {"type": "export", "target": "test"} , "test": { "type": "install" , "deps": [["TREE", null, "src"]] } } EOF echo echo "Creating repos.json at ${FOO_REPO}:" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "." , "pragma": {"to_git": true} } } } } EOF cat repos.json echo git init git checkout --orphan foomaster git config user.name 'Nobody' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo' 2>&1 # Set up repo bar readonly BAR_REPO="${REPO_DIRS}/bar" mkdir -p "${BAR_REPO}" cd "${BAR_REPO}" echo echo "Creating repos.template.json at ${BAR_REPO}:" cat > repos.template.json << EOF { "repositories": { "": { "repository": { "type": "tree structure" , "repo": "foo" } } } } EOF cat repos.template.json # Import foo to bar "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ 2>&1 > repos.json echo repos.json cat repos.json echo git init git checkout --orphan barmaster git config user.name 'Nobody' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar' 2>&1 # Set up main repo readonly MAIN_ROOT="${REPO_DIRS}/main" mkdir -p "${MAIN_ROOT}" cd "${MAIN_ROOT}" touch ROOT echo echo "Creating TARGETS at ${MAIN_ROOT}:" cat > TARGETS << 'EOF' { "": {"type": "export", "target": "gen"} , "gen": { "type": "generic" , "cmds": ["ls -R src > out.txt"] , "outs": ["out.txt"] , "deps": [["TREE", null, "src"]] } } EOF cat TARGETS echo echo "Creating TARGETS.result at ${MAIN_ROOT}:" cat > TARGETS.result << 'EOF' { "rename_1": { "type": "generic" , "cmds": ["mv out.txt foo.txt"] , "outs": ["foo.txt"] , "deps": [["@", "structure_1", "", ""]] } , "rename_2": { "type": "generic" , "cmds": ["mv out.txt bar.txt"] , "outs": ["bar.txt"] , "deps": [["@", "structure_2", "", ""]] } , "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": ["rename_1", "rename_2"] } } EOF cat TARGETS.result echo echo "Creating repos.template.json at ${MAIN_ROOT}:" cat > repos.template.json << EOF { "repositories": { "targets": { "repository": { "type": "file" , "path": "." , "pragma": {"to_git": true} } } , "structure": { "repository": "bar" , "target_root": "targets" } , "structure_2": { "repository": { "type": "tree structure" , "repo": "foo" } , "target_root": "targets" } , "result": { "repository": { "type": "file" , "path": "." , "pragma": {"to_git": true} } , "target_root": "targets" , "target_file_name": "TARGETS.result" , "bindings": { "structure_1": "structure" , "structure_2": "structure_2" } } } } EOF cat repos.template.json "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" \ 2>&1 > repos-full.json echo repos-full.json cat repos-full.json # Dump the graph before deduplication: echo "${JUST_MR}" -C repos-full.json --norc --just "${JUST}" \ --local-build-root "${LBR}" --main "result" \ -L '["env", "PATH='"${PATH}"'"]' analyse \ --dump-plain-graph actions-full.json 2>&1 # Run deduplication: echo cat repos-full.json | "${DEDUPLICATE}" 2>&1 > repos.json cat repos.json echo # Dump the graph after deduplication: "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" --main "result" \ -L '["env", "PATH='"${PATH}"'"]' analyse \ --dump-plain-graph actions.json 2>&1 # Verify that we reduced the number of repositories, but did # not change the action graph (except for the origins of the actions). [ $(jq -aM '.repositories | length' repos.json) -lt $(jq -aM '.repositories | length' repos-full.json) ] cmp actions-full.json actions.json # Check the result can be built after deduplication: echo "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" --main "result" \ -L '["env", "PATH='"${PATH}"'"]' install \ -o "${OUT}/result" 2>&1 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/000077500000000000000000000000001516554100600244435ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/TARGETS000066400000000000000000000105431516554100600255020ustar00rootroot00000000000000{ "git-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-imports"] , "test": ["git-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "git-import-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "deduplicate": { "type": ["@", "rules", "shell/test", "script"] , "name": ["deduplicate"] , "test": ["deduplicate.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "deduplicate-tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] , "keep": [ "repos-keep.json" , "actions-keep.json" , "repos.json" , "actions.json" , "repos-dedup.json" ] } , "absent": { "type": ["@", "rules", "shell/test", "script"] , "name": ["absent"] , "test": ["absent.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "computed": { "type": ["@", "rules", "shell/test", "script"] , "name": ["computed"] , "test": ["computed.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] , "keep": ["repos.json", "actions.json"] } , "file-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["file-imports"] , "test": ["file-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "archive-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["archive-imports"] , "test": ["archive-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "git-tree-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-tree-imports"] , "test": ["git-tree-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "plain-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["plain-imports"] , "test": ["plain-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "generic-imports": { "type": ["@", "rules", "shell/test", "script"] , "name": ["generic-imports"] , "test": ["generic-imports.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "git-import-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "clone-file": { "type": ["@", "rules", "shell/test", "script"] , "name": ["clone-file"] , "test": ["clone/file-repos.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "clone-git": { "type": ["@", "rules", "shell/test", "script"] , "name": ["clone-git"] , "test": ["clone/git-repos.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "clone-archive": { "type": ["@", "rules", "shell/test", "script"] , "name": ["clone-archive"] , "test": ["clone/archive-repos.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] , ["end-to-end/just-mr", "create_test_archives"] ] } , "clone-git-tree": { "type": ["@", "rules", "shell/test", "script"] , "name": ["clone-git-tree"] , "test": ["clone/git-tree-repos.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end", "lock-tool-under-test"] ] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "stage": ["just-lock"] , "deps": { "type": "++" , "$1": [ ["deduplicate", "absent"] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": [ "git-imports" , "computed" , "file-imports" , "archive-imports" , "git-tree-imports" , "plain-imports" , "generic-imports" , "clone-file" , "clone-git" , "clone-archive" , "clone-git-tree" ] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/absent.sh000066400000000000000000000065401516554100600262600ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo/" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": {"repository": {"type": "file", "path": "src"}, "bindings": {"bar": "bar"}} , "bar": { "repository": { "type": "archive" , "content": "4d86bdf4d07535db3754e6cbf30a6fc81aaa2cc1" , "fetch": "https://example.org/v1.2.3" } } } } EOF git init 2>&1 git checkout --orphan foomaster 2>&1 git config user.name 'N.O.Body' 2>&1 git config user.email 'nobody@example.org' 2>&1 git add . 2>&1 git commit -m 'Add foo' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.in.json <&1 echo cat repos.json echo # Both bar repos should be unified by deduplication FOO_BAR=$(jq '.repositories.foo.bindings.bar' repos.json) echo "Bar repo of foo: ${FOO_BAR}" INTERNAL_BAR=$(jq '.repositories."".bindings.bar' repos.json) echo "Internal bar repo: ${INTERNAL_BAR}" [ "${FOO_BAR}" = "${INTERNAL_BAR}" ] jq '.repositories.'"$FOO_BAR" repos.json > bar.json cat bar.json # ... and the pragma empty, as the internal bar is part of the merge # and that repository cannot be absent. [ $(jq '.pragma?' bar.json) = "null" ] # Also, both file repositories should be unified FILE_FOO=$(jq '.repositories."".bindings."file-foo"' repos.json) echo "file-foo of '' is now ${FILE_FOO}" FILE_BAR=$(jq '.repositories."".bindings."file-bar"' repos.json) echo "file-bar of '' is now ${FILE_BAR}" [ "${FILE_FOO}" = "${FILE_BAR}" ] jq '.repositories.'"$FILE_FOO" repos.json > local.json cat local.json # The to_git pragma should be preserved [ $(jq '.repository.pragma."to_git"' local.json) = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/archive-imports.sh000066400000000000000000000070621516554100600301200ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOCK_LBR="${TEST_TMPDIR}/lock-local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" mkdir -p "${DISTDIR}" # Repo foo mkdir -p "${REPO_DIRS}/foo/root/src/inner" cd "${REPO_DIRS}/foo" cat > root/repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"special": "resolve-completely"} } } } } EOF # add symlink to check that special pragma is needed and is inherited in import ln -s inner/actual_TARGETS root/src/TARGETS cat > root/src/inner/actual_TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF tar cf "${DISTDIR}/foo-1.2.3.tar" . 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" # Repo bar mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": ""}}}} EOF cat > TARGETS <<'EOF' { "": {"type": "file_gen", "name": "bar.txt", "data": "BAR"}} EOF tar cf "${DISTDIR}/bar-1.2.3.tar" . 2>&1 barcontent=$(git hash-object "${DISTDIR}/bar-1.2.3.tar") echo "Bar archive has content ${barcontent}" # As we do not have an actual remote server to host the archives, make them # available in local CAS instead HASHFOO=$("${JUST}" add-to-cas --local-build-root "${LOCK_LBR}" "${DISTDIR}/foo-1.2.3.tar") [ "${HASHFOO}" = "${foocontent}" ] HASHBAR=$("${JUST}" add-to-cas --local-build-root "${LOCK_LBR}" "${DISTDIR}/bar-1.2.3.tar") [ "${HASHBAR}" = "${barcontent}" ] # Main repo mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF # Import repos as archives cat > repos.in.json <&1 cat repos.json echo "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ --distdir "${DISTDIR}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q FOOBAR "${OUT}/out.txt" echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/clone/000077500000000000000000000000001516554100600255435ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/clone/archive-repos.sh000066400000000000000000000112261516554100600306500ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LOCK_LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_ARCHIVES="${TEST_TMPDIR}/local-build-root-archives" readonly LBR_CLONES="${TEST_TMPDIR}/local-build-root-clones" readonly INSTALL_ZIP="${TEST_TMPDIR}/install-zip" readonly INSTALL_TGZ="${TEST_TMPDIR}/install-tgz" readonly INSTALL_FOREIGN="${TEST_TMPDIR}/install-foreign" readonly INSTALL_DISTDIR="${TEST_TMPDIR}/install-distdir" readonly ROOT=`pwd` readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly WRKDIR="${TEST_TMPDIR}/work" # Set up zip and tgz archives mkdir -p "${DISTDIR}" cd "${DISTDIR}" ${ROOT}/src/create_test_archives readonly ZIP_REPO_CONTENT=$("${JUST}" add-to-cas --local-build-root "${LOCK_LBR}" "${DISTDIR}/zip_repo.zip") readonly TGZ_REPO_CONTENT=$("${JUST}" add-to-cas --local-build-root "${LOCK_LBR}" "${DISTDIR}/tgz_repo.tar.gz") # Set up foreign file echo Here be dragons > "${DISTDIR}/foreign.txt" readonly FOREIGN=$("${JUST}" add-to-cas --local-build-root "${LOCK_LBR}" "${DISTDIR}/foreign.txt") # Input configuration file mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.in.json <&1 echo echo Clone repos: CLONE_TO="cloned_foo" cat > clone.json <&1 echo echo Output config: cat repos.json echo echo Check output configuration: grep "${CLONE_TO}/zip" repos.json grep "${CLONE_TO}/tgz" repos.json grep "${CLONE_TO}/foreign" repos.json grep "${CLONE_TO}/distdir" repos.json # check special pragmas are kept grep resolve-completely repos.json grep ignore repos.json echo # Check setup with local clones: "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ -C repos.json --local-build-root "${LBR_CLONES}" setup --all 2>&1 echo # Check that the clones have the expected content [ $(jq '."repositories"."zip_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \ = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" --resolve-special=tree-all "${CLONE_TO}/zip") ] [ $(jq '."repositories"."tgz_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \ = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" --resolve-special=ignore "${CLONE_TO}/tgz") ] [ $(jq '."repositories"."foreign_file"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \ = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" "${CLONE_TO}/foreign") ] [ $(jq '."repositories"."distdir_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \ = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" "${CLONE_TO}/distdir") ] echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/clone/file-repos.sh000066400000000000000000000035631516554100600301530ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly FILE_ROOT="${TEST_TMPDIR}/file-root" readonly WRKDIR="${PWD}/work" # Set up file repo mkdir -p "${FILE_ROOT}/test_dir" echo test > "${FILE_ROOT}/test_dir/test_file" ln -s test_file "${FILE_ROOT}/test_dir/test_link" # Main repository mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.in.json <&1 echo echo Output config: cat repos.json echo echo Check cloned entries: [ -f "${CLONE_TO}/test_file" ] && [ -L "${CLONE_TO}/test_link" ] || exit 1 grep test "${CLONE_TO}/test_file" readlink -m "${CLONE_TO}/test_link" | grep test_file echo echo Check output configuration: grep "${CLONE_TO}" repos.json grep "pragma" repos.json echo echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/clone/git-repos.sh000066400000000000000000000103171516554100600300120ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_1="${TEST_TMPDIR}/local-build-root-1" readonly LBR_2="${TEST_TMPDIR}/local-build-root-2" readonly OUT_1="${TEST_TMPDIR}/build-output-1" readonly OUT_2="${TEST_TMPDIR}/build-output-2" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" # Repository 'foo' mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"special": "ignore"} } } } } EOF cat > src/TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF # add symlink to check that special pragma is needed and is inherited in import ln -s ../../../nonexistent src/to_ignore git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 # Repository 'bar' depending on 'foo' mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.in.json <&1 cat repos.json echo cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > foobar.txt"] , "outs": ["foobar.txt"] , "deps": [["@", "foo", "", ""], "internal"] } , "internal": {"type": "file_gen", "name": "bar.txt", "data": "BAR"} } EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 # Main repository depending on 'bar' mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foobar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "bar", "", ""]] } } EOF cat > repos.in.json <&1 cat repos.json echo echo echo Check main repo build: "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ --local-build-root "${LBR_1}" install -o "${OUT_1}" 2>&1 echo cat "${OUT_1}/out.txt" echo grep -q FOOBAR "${OUT_1}/out.txt" echo echo Clone dependency foo of repository bar: CLONE_TO="cloned_foo" "${JUST_LOCK}" -C repos.in.json -o repos.json \ --clone '{"'${CLONE_TO}'": ["bar", ["foo"]]}' \ --local-build-root "${LBR}" 2>&1 echo cat repos.json echo echo Check output configuration: grep "${CLONE_TO}" repos.json grep pragma repos.json [ -f "${CLONE_TO}/repos.json" ] && [ -d "${CLONE_TO}/src" ] \ && [ -f "${CLONE_TO}/src/TARGETS" ] && [ -L "${CLONE_TO}/src/to_ignore" ] || exit 1 echo echo Check build with cloned repo: "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ --local-build-root "${LBR_2}" install -o "${OUT_2}" 2>&1 echo cat "${OUT_1}/out.txt" echo grep -q FOOBAR "${OUT_1}/out.txt" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/clone/git-tree-repos.sh000066400000000000000000000054521516554100600307530ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LOCK_LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_1="${TEST_TMPDIR}/local-build-root-1" readonly LBR_2="${TEST_TMPDIR}/local-build-root-2" readonly SCRATCH_LBR="${TEST_TMPDIR}/scratch-local-build-root" readonly SCRATCH="${TEST_TMPDIR}/scratch-space" readonly ROOT=`pwd` readonly BIN="${TEST_TMPDIR}/bin" readonly WRKDIR="${TEST_TMPDIR}/work" # Set up git tree mkdir -p "${BIN}" cat > "${BIN}/mock-vcs" <<'EOF' #!/bin/sh mkdir -p foo/bar/baz echo foo data > foo/data.txt echo bar data > foo/bar/data.txt echo baz data > foo/bar/baz/data.txt EOF chmod 755 "${BIN}/mock-vcs" export PATH="${BIN}:${PATH}" rm -rf "${SCRATCH}" mkdir -p "${SCRATCH}" (cd "${SCRATCH}" && "${BIN}/mock-vcs") FOREIGN_GIT_TREE=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" "${SCRATCH}") # Input configuration file mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.in.json <&1 echo [ "$(jq '."repositories"."git_tree"."workspace_root" | .[1]' "${CONF}")" \ = \""${FOREIGN_GIT_TREE}"\" ] echo Clone repo: CLONE_TO="cloned_foo" "${JUST_LOCK}" -C repos.in.json -o repos.json --local-build-root "${LOCK_LBR}" \ --clone '{"'${CLONE_TO}'": ["git_tree", []]}' 2>&1 echo echo Output config: cat repos.json echo echo Check output configuration: grep "${CLONE_TO}" repos.json echo # Check setup with local clones: "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ -C repos.json --local-build-root "${LBR_2}" setup 2>&1 echo # Check clone location has the expected content [ $("${JUST}" add-to-cas --local-build-root "${LBR_2}" "${CLONE_TO}") \ = "${FOREIGN_GIT_TREE}" ] echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/computed.sh000066400000000000000000000126071516554100600266250ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}" # Set up repo foo mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"to_git": true} } } } } EOF cat > src/TARGETS <<'EOF' { "": {"type": "export", "target": "gen"} , "gen": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo -n FOO > foo.txt"]} } EOF git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 # Set up repo bar mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.in.json <&1 echo cat repos.json echo cat > generate.py <<'EOF' import json import sys COUNT = int(sys.argv[1]) targets = {} for i in range(COUNT): targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], "cmds": ["seq 0 %d > %d.txt" % (i, i)]} targets[""] = {"type": "export", "target": "gen"} targets["gen"] = {"type": "generic", "deps": ["%d" % i for i in range(COUNT)], "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + ["> out"])], "outs": ["out"]} print (json.dumps(targets, indent=2)) EOF cat > TARGETS <<'EOF' { "": {"type": "export", "flexible_config": ["COUNT"], "target": "generate"} , "generate": { "type": "generic" , "arguments_config": ["COUNT"] , "outs": ["TARGETS", "bar.txt"] , "deps": ["generate.py", ["@", "foo", "", ""]] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "python3" , "generate.py" , {"type": "var", "name": "COUNT"} , ">" , "TARGETS" ] } , "cat foo.txt | tr A-Z a-z > bar.txt" ] } } EOF mkdir src cp generate.py src/generate.py cp TARGETS src/TARGETS git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 # Set up repo to build mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": {"type": "export", "target": "gen"} , "gen": { "type": "generic" , "cmds": ["cat bar.txt init.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "bar_root", "", ""], "init"] } , "init": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > init.txt"] , "outs": ["init.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.in.json <&1 echo cat repos.json echo grep DoNotImport && exit 1 || : # we should not bring in unneeded bindings "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ -L '["env", "PATH='"${PATH}"'"]' 2>&1 echo echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/deduplicate.sh000077500000000000000000000106231516554100600272670ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly DEDUPLICATE="${PWD}/bin/deduplicate-tool-under-test" readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "src"}}}} EOF cat > src/TARGETS <<'EOF' { "": {"type": "generic", "outs": ["foo.txt"], "cmds": ["echo -n FOO > foo.txt"]} } EOF git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.in.json <&1 echo cat repos.json echo cat > TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["bar.txt"] , "cmds": ["cat foo.txt | tr A-Z a-z > bar.txt"] , "deps": [["@", "foo", "", ""]] } } EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF # Check "keep" field works by keeping both version of foo (direct and transient) cat > repos.in.json <&1 echo cat repos-keep.json echo "${JUST_MR}" -C repos-keep.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ --dump-plain-graph actions-keep.json 2>&1 echo # Check that deduplication (performed by default) works as expected cat > repos.in.json <&1 echo cat repos.json echo "${JUST_MR}" -C repos.json --norc --just "${JUST}" \ --local-build-root "${LBR}" analyse \ --dump-plain-graph actions.json 2>&1 echo # Check against existing tooling cat repos-keep.json | "${DEDUPLICATE}" > repos-dedup.json jq . repos.json > file_a.json jq . repos-dedup.json > file_b.json cmp file_a.json file_b.json # Verify that deduplication reduces the number of repositories, but does # not change the action graph (except for the origins of the actions). [ $(jq -aM '.repositories | length' "repos.json") -lt $(jq -aM '.repositories | length' "repos-keep.json") ] cmp actions-keep.json actions.json echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/file-imports.sh000066400000000000000000000055411516554100600274160ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" mkdir -p "${REPO_DIRS}/foo/src/inner" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"special": "resolve-completely"} } } } } EOF # add symlink to check that special pragma is needed and is inherited in import ln -s inner/actual_TARGETS src/TARGETS cat > src/inner/actual_TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": ""}}}} EOF cat > TARGETS <<'EOF' { "": {"type": "file_gen", "name": "bar.txt", "data": "BAR"}} EOF mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF # Import repos as local checkouts cat > repos.in.json <&1 cat repos.json echo # Check pragmas for repo "foo" are correctly set [ $(jq -r '.repositories.foo.repository.pragma.special' repos.json) = "resolve-completely" ] [ $(jq -r '.repositories.foo.repository.pragma.to_git' repos.json) = true ] # Check the symlink gets resolved as expected "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q FOOBAR "${OUT}/out.txt" echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/generic-imports.sh000066400000000000000000000070701516554100600301120ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" # make tools available in PATH export PATH="$(pwd)/bin:${PATH}" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"special": "ignore"} } } } } EOF cat > src/TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF # add symlink to check that special pragma is needed and is inherited in import ln -s ../../../nonexistent src/causes_fail git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": ""}}}} EOF cat > TARGETS <<'EOF' { "": {"type": "file_gen", "name": "bar.txt", "data": "BAR"}} EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.in.json <&1 echo cat repos.json echo "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q FOOBAR "${OUT}/out.txt" # Verify that results match a chained import, as no deduplication is to be done cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo", "bar": "bar"} } } } EOF "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" > repos-chained.json jq . repos.json > repos.json jq . repos-chained.json > repos-chained.json diff repos.json repos-chained.json echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/git-imports.sh000066400000000000000000000067041516554100600272640ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly GIT_IMPORT="${PWD}/bin/git-import-under-test" readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" mkdir -p "${REPO_DIRS}/foo/src" cd "${REPO_DIRS}/foo" cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "src" , "pragma": {"special": "ignore"} } } } } EOF cat > src/TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "FOO"}} EOF # add symlink to check that special pragma is needed and is inherited in import ln -s ../../../nonexistent src/causes_fail git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add foo.txt' 2>&1 mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": ""}}}} EOF cat > TARGETS <<'EOF' { "": {"type": "file_gen", "name": "bar.txt", "data": "BAR"}} EOF git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add . git commit -m 'Add bar.txt' 2>&1 mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt bar.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""], ["@", "bar", "", ""]] } } EOF cat > repos.in.json <&1 echo cat repos.json echo "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q FOOBAR "${OUT}/out.txt" # Verify that results match a chained import, as no deduplication is to be done cat > repos.template.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"foo": "foo", "bar": "bar"} } } } EOF "${GIT_IMPORT}" -C repos.template.json --as foo -b foomaster "${REPO_DIRS}/foo" \ | "${GIT_IMPORT}" -C - --as bar -b barmaster "${REPO_DIRS}/bar" > repos-chained.json jq . repos.json > repos.json jq . repos-chained.json > repos-chained.json diff repos.json repos-chained.json echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/git-tree-imports.sh000066400000000000000000000101061516554100600302100ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR2="${TEST_TMPDIR}/local-build-root-2" readonly LBR3="${TEST_TMPDIR}/local-build-root-3" readonly LBR4="${TEST_TMPDIR}/local-build-root-4" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" # Git-tree generator and its own generator mkdir -p bin cat > bin/mock-vcs <<'EOF' #!/bin/sh if [ "$(cat ${CREDENTIAL_PATH:-/dev/null})" = "sEcReT" ] then mkdir -p data echo "$1" > data/sources.txt ln -s ../../../nonexistent data/causes_fail echo '{"":{"type":"install","dirs":[[["TREE",null,"."],"."]]}}' > data/TARGETS echo '{"repositories":{"":{"repository":{"type":"file","path":".","pragma":{"special":"ignore"}}}}}' > data/repos.json else echo 'not enough credentials available' fi EOF chmod 755 bin/mock-vcs cat > bin/mock-vcs-gen <<'EOF' #!/bin/sh echo '["mock-vcs", "checkout"]' EOF chmod 755 bin/mock-vcs-gen export PATH="$(pwd)/bin:${PATH}" mkdir -p etc echo -n sEcReT > etc/pass export CREDENTIAL_PATH="$(pwd)/etc/pass" # Compute tree of our mock checkout mkdir -p temp ( cd temp mock-vcs "checkout" 2>&1 git init 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Sample output" 2>&1 ) readonly TREE_ID=$(cd temp && git log -n 1 --format="%T") echo "Tree of checkout is ${TREE_ID}." readonly SUBTREE_ID=$(cd temp && git rev-parse "${TREE_ID}":data) echo "Tree of data is ${SUBTREE_ID}" rm -rf temp # Main repo mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat sources.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""]] } } EOF cat > repos.in.json <&1 cat repos.json echo # Check the expected Git tree identifier of the subdir in the configuration grep -q "${SUBTREE_ID}" repos.json # Check command generation cat > repos.in.json <&1 cat repos-gen.json echo # Check the same Git tree identifier of the subdir is in the configuration grep -q "${SUBTREE_ID}" repos-gen.json # Check successful build "${JUST_MR}" --norc -L '["env", "PATH='"${PATH}"'"]' --just "${JUST}" \ --local-build-root "${LBR3}" install -o "${OUT}" 2>&1 echo grep checkout "${OUT}/out.txt" echo # Verify the environment is needed export CREDENTIAL_PATH=/dev/null "${JUST_MR}" --norc -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBR4}" setup 2>&1 && exit 1 || : echo echo "failed as expected" echo echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-lock/plain-imports.sh000066400000000000000000000070501516554100600275770ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_PLAIN="${TEST_TMPDIR}/local-build-root-plain" readonly OUT="${TEST_TMPDIR}/build-output" readonly OUT_PLAIN="${TEST_TMPDIR}/build-output-plain" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" mkdir -p "${REPO_DIRS}/foo/inner" cd "${REPO_DIRS}/foo" touch ROOT cat > repos.json <<'EOF' { "repositories": { "": { "repository": { "type": "file" , "path": "inner" } } } } EOF # add resolvable linked TARGETS file ln -s inner/linked_TARGETS TARGETS cat > inner/linked_TARGETS <<'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "LINK"}} EOF # add inner TARGETS file shadowed by the linked one cat > inner/TARGETS << 'EOF' { "": {"type": "file_gen", "name": "foo.txt", "data": "INNER"}} EOF mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["cat foo.txt > out.txt"] , "outs": ["out.txt"] , "deps": [["@", "foo", "", ""]] } } EOF echo === Check normal import === cat > repos.in.json <&1 cat repos.json echo # Check pragmas: "to_git" is kept [ $(jq -r '.repositories.foo.repository.pragma.to_git' repos.json) = true ] # Check that the subdir is taken as expected "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ --local-build-root "${LBR}" install -o "${OUT}" 2>&1 echo cat "${OUT}/out.txt" echo grep -q INNER "${OUT}/out.txt" echo == Check plain import === cat > repos.in.json <&1 cat repos.json echo # Check pragmas: "to_git" is kept, "special" is unconditionally set [ $(jq -r '.repositories.foo.repository.pragma.special' repos.json) = "resolve-completely" ] [ $(jq -r '.repositories.foo.repository.pragma.to_git' repos.json) = true ] # Check the symlink gets resolved as expected "${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ --local-build-root "${LBR_PLAIN}" install -o "${OUT_PLAIN}" 2>&1 echo cat "${OUT_PLAIN}/out.txt" echo grep -q LINK "${OUT_PLAIN}/out.txt" echo "OK" just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/000077500000000000000000000000001516554100600241315ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/TARGETS000066400000000000000000000232171516554100600251720ustar00rootroot00000000000000{ "cas-independent": { "type": ["@", "rules", "shell/test", "script"] , "name": ["cas-independent"] , "test": ["cas-independent.sh"] , "deps": [["", "mr-tool-under-test"]] } , "fetch": { "type": ["@", "rules", "shell/test", "script"] , "name": ["fetch"] , "test": ["fetch.sh"] , "deps": [["", "mr-tool-under-test"]] } , "fetch-gc": { "type": ["@", "rules", "shell/test", "script"] , "name": ["fetch-gc"] , "test": ["fetch-gc.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "create_test_archives": { "type": ["@", "rules", "CC", "binary"] , "tainted": ["test"] , "name": ["create_test_archives"] , "srcs": ["create_test_archives.cpp"] , "private-deps": [ ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/utils/archive", "archive_ops"] , ["@", "src", "src/utils/cpp", "tmp_dir"] ] , "private-ldflags": ["-pthread"] , "stage": ["src"] } , "just_mr_mp": { "type": ["@", "rules", "shell/test", "script"] , "name": ["just_mr_mp"] , "test": ["just-mr.test.sh"] , "deps": [ "create_test_archives" , ["", "mr-tool-under-test"] , ["utils", "test_utils_install"] ] } , "just_mr_mirrors": { "type": ["@", "rules", "shell/test", "script"] , "name": ["just_mr_mirrors"] , "test": ["just-mr-mirrors.test.sh"] , "deps": [ "create_test_archives" , ["", "mr-tool-under-test"] , ["utils", "null server"] , ["utils", "test_utils_install"] ] } , "git-tree-verbosity": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-tree-verbosity"] , "test": ["verbosity.sh"] , "deps": [["", "mr-tool-under-test"]] } , "defaults": { "type": ["@", "rules", "shell/test", "script"] , "name": ["default-values"] , "test": ["defaults.sh"] , "deps": [["", "mr-tool-under-test"]] } , "install-roots": { "type": ["@", "rules", "shell/test", "script"] , "name": ["install-roots"] , "test": ["install-roots.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "install-roots-symlinks": { "type": ["@", "rules", "shell/test", "script"] , "name": ["install-roots-symlinks"] , "test": ["install-roots-symlinks.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["log/archive.txt", "log/file.txt"] } , "fetch-remote": { "type": ["end-to-end", "with remote"] , "name": ["fetch-remote"] , "test": ["fetch-remote.sh"] , "deps": [["", "mr-tool-under-test"]] } , "fetch-remote-git-tree": { "type": ["end-to-end", "with remote"] , "name": ["fetch-remote-git-tree"] , "test": ["fetch-remote-git-tree.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "absent-roots": { "type": ["@", "rules", "shell/test", "script"] , "name": ["absent-roots"] , "test": ["absent-roots.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "fetch-absent": { "type": ["end-to-end", "with serve"] , "name": ["fetch-absent"] , "test": ["fetch-absent.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["fetch-absent (data)"] } , "fetch-absent-git-tree": { "type": ["end-to-end", "with serve"] , "name": ["fetch-absent-git-tree"] , "test": ["fetch-absent-git-tree.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["fetch-absent (data)"] } , "absent-closure": { "type": ["@", "rules", "shell/test", "script"] , "name": ["absent-closure"] , "test": ["absent-closure.sh"] , "deps": [["", "mr-tool-under-test"]] } , "absent-config": { "type": ["end-to-end", "with serve"] , "name": ["absent-config"] , "test": ["absent-config.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["fetch-absent (data)"] } , "fetch-absent-archives": { "type": ["end-to-end", "with serve"] , "name": ["fetch-absent-archives"] , "test": ["fetch-absent-archives.sh"] , "deps": [ "fetch-absent (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["fetch-absent (data)"] } , "fetch-absent-archives-symlinks": { "type": ["end-to-end", "with serve"] , "name": ["fetch-absent-archives-symlinks"] , "test": ["fetch-absent-archives-symlinks.sh"] , "deps": [ "fetch-absent-with-symlinks (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["fetch-absent-with-symlinks (data)"] } , "fetch-absent-distdir-archive": { "type": ["end-to-end", "with serve"] , "name": ["fetch-absent-distdir-archive"] , "test": ["fetch-absent-distdir-archive.sh"] , "deps": [ "fetch-absent (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["fetch-absent (data)"] } , "fetch-absent (data)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": [ "mkdir -p src" , "for i in `seq 1 10` ; do echo $i > src/$i.txt ; done" , "tar cf src/data.tar src/*.txt" ] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "fetch-absent-with-symlinks (data)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": [ "mkdir -p src/foo" , "echo x > src/foo/data.txt" , "ln -s foo/data.txt src/link" , "tar cf src/data.tar src/*" ] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "git-tree-env": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-tree-git-tree-env"] , "test": ["git-tree-env.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "git-env": { "type": ["@", "rules", "shell/test", "script"] , "name": ["git-env"] , "test": ["git-env.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "foreign-file": { "type": ["@", "rules", "shell/test", "script"] , "name": ["foreign-file"] , "test": ["foreign-file.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "reporting-verbosity": { "type": ["end-to-end", "with remote"] , "name": ["reporting-verbosity"] , "test": ["reporting-verbosity.sh"] , "deps": [ "fetch-absent (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] } , "stay-local": { "type": ["end-to-end", "with remote"] , "name": ["stay-local"] , "test": ["stay-local.sh"] , "deps": [ "fetch-absent (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["utils", "null server"] ] } , "gc-repo": { "type": ["@", "rules", "shell/test", "script"] , "name": ["gc-repo"] , "test": ["gc-repo.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["log/log-1", "log/log-2"] } , "repeated-gc": { "type": ["@", "rules", "shell/test", "script"] , "name": ["repeated-gc"] , "test": ["repeated-gc.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "gc-drop": { "type": ["@", "rules", "shell/test", "script"] , "name": ["gc-drop"] , "test": ["gc-drop.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "ignore-special": { "type": ["@", "rules", "shell/test", "script"] , "name": ["ignore-special"] , "test": ["ignore-special.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "invocation-log": { "type": ["@", "rules", "shell/test", "script"] , "name": ["invocation-log"] , "test": ["invocation-log.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] , "keep": ["out/just-repo-config.json"] } , "cas-root": { "type": ["end-to-end", "with serve"] , "name": ["cas-root"] , "test": ["cas-root.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["lbr-analyse", "out"] } , "error-reporting": { "type": ["@", "rules", "shell/test", "script"] , "name": ["error-reporting"] , "test": ["error-reporting.sh"] , "deps": [["", "mr-tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR", "TEST_COMPATIBLE_REMOTE"] , "stage": ["just-mr"] , "deps": { "type": "++" , "$1": [ ["cas-independent", "fetch", "fetch-gc", "install-roots"] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": { "type": "++" , "$1": [ [ "install-roots-symlinks" , "just_mr_mp" , "just_mr_mirrors" , "git-tree-verbosity" , "git-tree-env" , "git-env" , "defaults" , "absent-roots" , "foreign-file" , "reporting-verbosity" , "gc-repo" , "repeated-gc" , "gc-drop" , "invocation-log" , "error-reporting" ] , { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": [] , "else": [ "fetch-remote" , "fetch-remote-git-tree" , "stay-local" , "cas-root" ] } , [ "fetch-absent" , "fetch-absent-git-tree" , "absent-closure" , "absent-config" , "fetch-absent-archives" , "fetch-absent-archives-symlinks" , "fetch-absent-distdir-archive" , "ignore-special" ] ] } } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/absent-closure.sh000066400000000000000000000053771516554100600274270ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # Test that users are warned if a non-content-fixed repository is reached from # an absent main repository. ## set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly FILE_ROOT="${TEST_TMPDIR}/file-repos" readonly TEST_ROOT="${TEST_TMPDIR}/test-root" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" mkdir -p "${OUT}" # Set up file repos mkdir -p "${FILE_ROOT}/test_dir1" echo test > "${FILE_ROOT}/test_dir1/test_file" mkdir -p "${FILE_ROOT}/test_dir2/inner_dir" echo test > "${FILE_ROOT}/test_dir2/inner_dir/test_file" mkdir -p "${FILE_ROOT}/test_dir3/inner_dir" echo test > "${FILE_ROOT}/test_dir3/inner_dir/test_file" # Setup sample repository config mkdir -p "${TEST_ROOT}" cd "${TEST_ROOT}" touch ROOT cat > repos.json <&1 echo echo Check expected warnings echo grep 'WARN.* "main" .*absent.* "main"' "${OUT}/log.txt" grep 'WARN.* "file_repo_not_content_fixed" .*absent.* "main"' "${OUT}/log.txt" grep 'WARN.* "file_repo_to_git_explicit" .*absent.* "main"' "${OUT}/log.txt" && echo "should have failed" && exit 1 || : grep 'WARN.* "file_repo_to_git_implicit" .*absent.* "main"' "${OUT}/log.txt" && echo "should have failed" && exit 1 || : echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/absent-config.sh000066400000000000000000000062461516554100600272140ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly ROOT="${PWD}" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly RCFILE="${TEST_TMPDIR}/mrrc.json" readonly OUT="${TEST_TMPDIR}/out" readonly LOCAL_REPO="${TEST_TMPDIR}/local-repository" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir -p "${LOCAL_REPO}" cd "${LOCAL_REPO}" mkdir src for c in a b c d e f g do echo $c > src/$c.txt done git init git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add -f . git commit -m 'Test repo' 2>&1 COMMIT_1=$(git log -n 1 --format="%H") mkdir -p "${ROOT}/work" cd "${ROOT}/work" touch ROOT cat > repos.json < main/TARGETS.A <<'EOF' { "": { "type": "generic" , "outs": ["outa.txt"] , "cmds": ["head -c 1 4.txt > outa.txt", "cat 2.txt >> outa.txt"] , "deps": ["4.txt", "2.txt"] } } EOF cat > main/TARGETS.B <<'EOF' { "": { "type": "generic" , "outs": ["outb.txt"] , "cmds": ["head -c 1 e.txt > outb.txt", "cat g.txt >> outb.txt"] , "deps": ["e.txt", "g.txt"] } } EOF cat > main/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["cat outa.txt outb.txt > out.txt"] , "deps": [["@", "A", "", ""], ["@", "B", "", ""]] } } EOF mkdir etc cat > etc/absent.json <<'EOF' ["A"] EOF cat > "${RCFILE}" <<'EOF' {"absent": [{"root": "workspace", "path": "etc/absent.json"}]} EOF echo echo CONF=$("${JUST_MR}" --local-build-root "${LBR}" \ --rc "${RCFILE}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" grep eg "${OUT}/out.txt" echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/absent-roots.sh000066400000000000000000000076521516554100600271170ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly INFOFILE="${PWD}/info.json" readonly PIDFILE="${PWD}/pid.txt" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly SERVE_LBR="${TEST_TMPDIR}/serve-local-build-root" readonly REPO_ROOT="${PWD}/serve_repo" readonly TEST_DATA="The content of the data file in foo" readonly LINK_TARGET="dummy" # Set up sample repository mkdir -p "${REPO_ROOT}/foo/bar" echo -n "${TEST_DATA}" > "${REPO_ROOT}/foo/bar/data.txt" ln -s "${LINK_TARGET}" "${REPO_ROOT}/foo/bar/link" { cd "${REPO_ROOT}" git init git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add -f . git commit -m 'Test repo' 2>&1 } readonly GIT_COMMIT_ID=$(cd "${REPO_ROOT}" && git log -n 1 --format='%H') readonly GIT_ROOT_ID=$(cd "${REPO_ROOT}" && git log -n 1 --format='%T') # Setup sample repository config touch ROOT cat > repos.json < .just-servec <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") cleanup() { kill $(cat "${PIDFILE}") } trap cleanup EXIT # Compute the repository configuration CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" --remote-serve-address localhost:${PORT} setup) cat "${CONF}" echo # Check that an absent root was created test "$(jq -r '.repositories.test.workspace_root[0]' "${CONF}")" = "git tree" test $(jq -r '.repositories.test.workspace_root | length' "${CONF}") = 2 # Check the tree was correctly set via 'just serve' remote test "$(jq -r '.repositories.test.workspace_root[1]' "${CONF}")" = "${GIT_ROOT_ID}" # Now check that an absent git tree repository works without running the command rm -rf "${LBR}" cat > repos.json < foo/bar/baz/data.txt echo "/baz/" > foo/bar/.gitignore # add a .gitignore tar cf "${DISTDIR}/foo-1.2.3.tar" foo 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" # get the tree identifier for later sanity check cd foo git init git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add -f . # add everything, including any .gitignore git commit -m 'Just care about the tree' 2>&1 tree_id=$(git log -n 1 --format='%T') cd .. rm -rf foo echo "foo as tree ${tree_id}" # Setup sample repository config touch ROOT cat > repos.json < $((4 * i)).dat echo 1 > $((4 * i + 1)).dat echo 0 > $((4 * i + 2)).dat echo 0 > $((4 * i + 3)).dat done echo "There NO reason whatsoever, that this file occurs bei by AcCiDeNt in local build root" > magic.txt DATA_TREE=$("${JUST}" add-to-cas --local-build-root "${LBR_COLLECT}" \ -r "${REMOTE_EXECUTION_ADDRESS}" .) MAGIC=$("${JUST}" add-to-cas --local-build-root "${LBR_COLLECT}" \ -r "${REMOTE_EXECUTION_ADDRESS}" magic.txt) echo "Data tree is ${DATA_TREE}, it contains a file with hash ${MAGIC}" # Locally we analyse the tree without fetching it mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > rc.json < repos.json < tools/TARGETS cat > tools/stats.py <<'EOF' #!/usr/bin/env python3 import json import os count = 0 sum = 0.0 sum_squares = 0.0 for root, dirs, files in os.walk('data'): for fname in files: if fname.endswith(".dat"): with open(os.path.join('data', fname)) as f: data = f.read() value = float(data) count += 1 sum += value sum_squares += value * value result = {"count": count, "sum x": sum, "sum x^2": sum_squares} with open("stats.json", "w") as f: json.dump(result, f) EOF chmod 755 tools/stats.py mkdir targets cat > targets/TARGETS <<'EOF' { "": {"type": "export", "target": "stats"} , "stats": { "type": "generic" , "outs": ["stats.json"] , "cmds": ["./stats.py"] , "deps": ["data", ["@", "tools", "", "stats.py"]] } , "data": {"type": "install", "dirs": [[["TREE", null, "."], "data"]]} } EOF "${JUST_MR}" --rc rc.json install -o "${OUT}" 2>&1 # Sanity check the result we obtained echo cat "${OUT}/stats.json" echo [ $(jq '.count | . == 80' "${OUT}/stats.json") = "true" ] [ $(jq '."sum x" | . == 20' "${OUT}/stats.json") = "true" ] [ $(jq '."sum x^2" | . == 20' "${OUT}/stats.json") = "true" ] # Verify that the root is not downloaded echo echo "Verifying ${MAGIC} is not known locally" "${JUST}" install-cas --local-build-root "${LBR_ANALYSE}" "${MAGIC}" 2>&1 && exit 1 || : echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/create_test_archives.cpp000066400000000000000000000137711516554100600310340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include // std::pair #include #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/archive/archive_ops.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { void SetupDefaultLogging() { LogConfig::SetLogLimit(LogLevel::Progress); LogConfig::SetSinks({LogSinkCmdLine::CreateFactory()}); } using file_t = std::pair; using filetree_t = std::unordered_map; /* Structure of content tree: +--root +--bar +--foo +--foo_l +--baz_l +--baz +--bar +--foo +--foo_l foo_l is symlink "baz/foo_l" [non-upwards, pointing to file] baz_l is symlink "baz" [non-upwards, pointing to tree] baz/foo_l is symlink "../foo_l" [upwards & confined, pointing to symlink] */ auto const kExpected = filetree_t{{"root", {"", ObjectType::Tree}}, {"root/foo", {"foo", ObjectType::File}}, {"root/bar", {"bar", ObjectType::File}}, {"root/baz", {"", ObjectType::Tree}}, {"root/baz_l", {"baz", ObjectType::Symlink}}, {"root/foo_l", {"foo", ObjectType::Symlink}}, {"root/baz/foo", {"foo", ObjectType::File}}, {"root/baz/bar", {"bar", ObjectType::File}}, {"root/baz/foo_l", {"../foo_l", ObjectType::Symlink}}}; void create_files(std::filesystem::path const& destDir = ".") { for (auto const& [path, file] : kExpected) { auto const& [content, type] = file; switch (type) { case ObjectType::File: { if (not FileSystemManager::WriteFile(content, destDir / path)) { Logger::Log(LogLevel::Error, "Could not create test file at path {}", (destDir / path).string()); std::exit(1); }; } break; case ObjectType::Tree: { if (not FileSystemManager::CreateDirectory(destDir / path)) { Logger::Log(LogLevel::Error, "Could not create test dir at path {}", (destDir / path).string()); std::exit(1); }; } break; case ObjectType::Symlink: { if (not FileSystemManager::CreateSymlink(content, destDir / path)) { Logger::Log(LogLevel::Error, "Could not create test symlink at path {}", (destDir / path).string()); std::exit(1); }; } break; default: { Logger::Log(LogLevel::Error, "File system failure in creating test archive"); std::exit(1); } } } } } // namespace /// \brief This code will create a zip and a tar.gz archives to be used in /// tests. The archives are created in a tmp directory and then posted in the /// current directory. Caller must guarantee write rights in current directory. auto main() -> int { // setup logging SetupDefaultLogging(); // make tmp dir auto curr_path = FileSystemManager::GetCurrentDirectory(); auto tmp_dir = TmpDir::Create(curr_path); if (not tmp_dir) { Logger::Log(LogLevel::Error, "Could not create tmp dir for test archives at path {}", curr_path.string()); return 1; } // create the content tree create_files(tmp_dir->GetPath()); // create the archives // 1. move to correct directory { auto anchor = FileSystemManager::ChangeDirectory(tmp_dir->GetPath()); // 2. create the archives wrt to current directory std::optional res{std::nullopt}; res = ArchiveOps::CreateArchive(ArchiveType::Zip, "zip_repo.zip", "root"); if (res) { Logger::Log(LogLevel::Error, "Creating test zip archive failed with:\n{}", *res); return 1; } res = ArchiveOps::CreateArchive( ArchiveType::TarGz, "tgz_repo.tar.gz", "root"); if (res) { Logger::Log(LogLevel::Error, "Creating test tar.gz archive failed with:\n{}", *res); return 1; } } // anchor released // 3. move archives to their correct location if (not FileSystemManager::Rename(tmp_dir->GetPath() / "zip_repo.zip", "zip_repo.zip")) { Logger::Log(LogLevel::Error, "Renaming zip archive failed"); return 1; } if (not FileSystemManager::Rename(tmp_dir->GetPath() / "tgz_repo.tar.gz", "tgz_repo.tar.gz")) { Logger::Log(LogLevel::Error, "Renaming tar.gz archive failed"); return 1; } // Done! Tmp dir will be cleaned automatically return 0; } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/defaults.sh000066400000000000000000000251071516554100600263010ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${TEST_TMPDIR}/put-log-files-here" mkdir -p "${LOG_DIR}" readonly RC_DIR="${TEST_TMPDIR}/etc/just-mr-rc-files" mkdir -p "${RC_DIR}" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" mkdir -p "${TOOLS_DIR}" readonly PARSE_DIR="${TEST_TMPDIR}/what-was-forwarded" mkdir -p "${PARSE_DIR}" readonly SAMPLE_RC="${RC_DIR}/sample.rc" cat > "${SAMPLE_RC}" < "${PARSE}" <<'EOF' #!/usr/bin/env python3 import json import os import sys from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("-C", dest="repository_config") parser.add_argument("-c", "--config", dest="build_config") parser.add_argument("--endpoint-configuration", dest="endpoint") parser.add_argument("-r", "--remote-execution-address", dest="remote_execution_address", default=None) parser.add_argument("--remote-instance-name", dest="remote_instance_name", default=None) parser.add_argument("--local-build-root", dest="local_build_root") parser.add_argument("-L","--local-launcher", dest="local_launcher") parser.add_argument("-f", "--log-file", dest="log_file", action="append", default=[]) parser.add_argument("--log-limit", dest="log_limit") parser.add_argument("--log-append", dest="log_append", action="store_true", default=False) parser.add_argument("-D", "--defines", dest="defines", action="append", default=[]) (options, args) = parser.parse_known_args(sys.argv[2:]) target_dir=args[-1] with open(os.path.join(target_dir, "defines"), "w") as f: f.write(json.dumps([json.loads(x) for x in options.defines])) with open(os.path.join(target_dir, "log-limit"), "w") as f: f.write(json.dumps(options.log_limit)) with open(os.path.join(target_dir, "log-file"), "w") as f: f.write(json.dumps(options.log_file)) if options.build_config: with open(os.path.join(target_dir, "config"), "w") as f: f.write(json.dumps(options.build_config)) else: if os.path.exists(os.path.join(target_dir, "config")): os.unlink(os.path.join(target_dir, "config")) if options.endpoint: with open(os.path.join(target_dir, "endpoint"), "w") as f: f.write(json.dumps(options.endpoint)) else: if os.path.exists(os.path.join(target_dir, "endpoint")): os.unlink(os.path.join(target_dir, "endpoint")) with open(os.path.join(target_dir, "launcher"), "w") as f: if options.local_launcher: f.write(options.local_launcher) else: f.write("null") with open(os.path.join(target_dir, "remote_execution_address"), "w") as f: f.write(json.dumps(options.remote_execution_address)) with open(os.path.join(target_dir, "remote_instance_name"), "w") as f: f.write(json.dumps(options.remote_instance_name)) EOF chmod 755 "${PARSE}" touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF ## log limit # rc is honored "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test "$(cat "${PARSE_DIR}/log-limit")" = '"4"' # command line overrides "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" --log-limit 5 build "${PARSE_DIR}" 2>&1 test "$(cat "${PARSE_DIR}/log-limit")" = '"5"' # value equal to default is discarded "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" --log-limit 3 build "${PARSE_DIR}" 2>&1 test "$(cat "${PARSE_DIR}/log-limit")" = 'null' ## log files # rc are taken rm -f "${LOG_DIR}/rc1.log" rm -f "${LOG_DIR}/rc2.log" "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test -f "${LOG_DIR}/rc1.log" test -f "${LOG_DIR}/rc2.log" test $(jq ". == [\"${LOG_DIR}/rc1.log\", \"${LOG_DIR}/rc2.log\"]" "${PARSE_DIR}/log-file") = "true" # command-line log files are added rm -f "${LOG_DIR}/rc1.log" rm -f "${LOG_DIR}/rc2.log" rm -f "${LOG_DIR}/cli.log" "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" -f "${LOG_DIR}/cli.log" build "${PARSE_DIR}" 2>&1 test -f "${LOG_DIR}/rc1.log" test -f "${LOG_DIR}/rc2.log" test -f "${LOG_DIR}/cli.log" test $(jq "sort == [\"${LOG_DIR}/cli.log\", \"${LOG_DIR}/rc1.log\", \"${LOG_DIR}/rc2.log\"]" "${PARSE_DIR}/log-file") = "true" ## launcher # rc is honored "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test $(jq '. == ["env", "SET_IN_RC=true"] ' "${PARSE_DIR}/launcher") = "true" # command-line takes precedence "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" -L '["cmd", "line", "launcher"]' \ build "${PARSE_DIR}" 2>&1 test $(jq '. == ["cmd", "line", "launcher"] ' "${PARSE_DIR}/launcher") = "true" # value equal to defaul is discared "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" -L '["env", "--"]' \ build "${PARSE_DIR}" 2>&1 test "$(cat "${PARSE_DIR}/launcher")" = 'null' ## Command-line -D # ignored on non-build commands "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${PARSE}" \ -D 'this is not json' version "${PARSE_DIR}" 2>&1 test $(jq '. == [] ' "${PARSE_DIR}/defines") = "true" # not forwarded, if empty "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${PARSE}" \ -D '{}' build "${PARSE_DIR}" 2>&1 test $(jq '. == [] ' "${PARSE_DIR}/defines") = "true" # combined on forwarding "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${PARSE}" \ -D '{"foo": "bar"}' -D '{"baz": "baz"}' -D '{"foo": "override"}' \ build "${PARSE_DIR}" 2>&1 test $(jq '. == [ {"foo": "override", "baz": "baz"}] ' "${PARSE_DIR}/defines") = true # but passed arguments are given separately "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${PARSE}" \ -D '{"foo": "bar"}' -D '{"baz": "baz"}' \ build -D '{"foo": "override"}' -D '{"x": "y"}' "${PARSE_DIR}" 2>&1 test $(jq '. == [ {"foo": "bar", "baz": "baz"}, {"foo": "override"}, {"x": "y"}] ' "${PARSE_DIR}/defines") = true ## -c # honored from rc-file if present touch sample-config.json "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/config" ] # not honored for non-analysing subcommands "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" install-cas "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/config" ] && exit 1 || : # not considered an error if not present rm -f sample-config.json "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/config" ] && exit 1 || : # not considered, if key not present in rc cat > tmprc.json <<'EOF' {"just files": {"unrelated": 123}} EOF "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc tmprc.json build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/config" ] && exit 1 || : ## --endpoint-configuration # honored from rc-file if present touch endpoint.json "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/endpoint" ] # not honored for non-endpoint-specific subcommands "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" install-cas "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/endpoint" ] && exit 1 || : # not considered an error if not present rm -f endpoint.json "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/endpoint" ] && exit 1 || : # not considered, if key not present in rc cat > tmprc.json <<'EOF' {"just files": {"unrelated": 123}} EOF "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc tmprc.json build "${PARSE_DIR}" 2>&1 [ -f "${PARSE_DIR}/endpoint" ] && exit 1 || : ## remote execution properties # rc-values are honored "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test $(jq '. == "127.0.0.1:1234"' "${PARSE_DIR}/remote_execution_address") = "true" test $(jq '. == "set/by/rc"' "${PARSE_DIR}/remote_instance_name") = "true" # CLI Override "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ -r 127.0.0.1:9999 --remote-instance-name "set/on/cli" \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test $(jq '. == "127.0.0.1:9999"' "${PARSE_DIR}/remote_execution_address") = "true" test $(jq '. == "set/on/cli"' "${PARSE_DIR}/remote_instance_name") = "true" # CLI Override, not setting instance name "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ -r 127.0.0.1:9999 \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test $(jq '. == "127.0.0.1:9999"' "${PARSE_DIR}/remote_execution_address") = "true" test $(jq '. == "set/by/rc"' "${PARSE_DIR}/remote_instance_name") = "true" # CLI Override with empty instance name "${JUST_MR}" --local-build-root "${LBR}" --just "${PARSE}" \ -r 127.0.0.1:9900 --remote-instance-name '' \ --rc "${SAMPLE_RC}" build "${PARSE_DIR}" 2>&1 test $(jq '. == "127.0.0.1:9900"' "${PARSE_DIR}/remote_execution_address") = "true" test $(jq '. == ""' "${PARSE_DIR}/remote_instance_name") = "true" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/error-reporting.sh000066400000000000000000000065341516554100600276350ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly BINDIR="${TEST_TMPDIR}/fake-bin" mkdir -p "${BINDIR}" mkdir work cd work touch ROOT echo echo '=== syntax error on CLI ===' exit_code=0 echo '{"this is not valid json"}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" --this-option-is-not-valid setup 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 67 ] echo echo '=== CLI OK, but config not JSON ===' exit_code=0 echo '{"this is not valid json"}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" setup 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 68 ] # config syntax error echo echo '=== everything syntactically OK, but: file repo, path missing ===' exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file"}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" setup 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 71 ] # error during setup echo echo '=== everything syntactically OK, but: file repo, path not a string ===' exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file", "path": 4711}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" setup 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 71 ] # error during setup echo echo '=== everything syntactically OK, but: file repo, path empty ===' exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file", "path": ""}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" setup 2>&1 || exit_code=$? echo "Exit code $exit_code" # The empty string is not a valid path, but it is OK to treat it as . [ '(' "$exit_code" -eq 71 ')' -o '(' "$exit_code" -eq 0 ')' ] echo echo '=== setup sucessfull, but fork failed ===' exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file", "path": "."}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${BINDIR}/does-not-exist" build 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 64 ] echo echo '=== non-zero exit code from build tool ===' cat > "${BINDIR}/return-42" <<'EOI' #!/bin/sh exit 42 EOI chmod 755 "${BINDIR}/return-42" exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file", "path": "."}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${BINDIR}/return-42" build 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 42 ] echo echo '=== exit code 0 from build tool ===' exit_code=0 echo '{"repositories": {"": {"repository": {"type": "file", "path": "."}}}}' > repos.json "${JUST_MR}" --norc --local-build-root "${LBR}" --just "true" build 2>&1 || exit_code=$? echo "Exit code $exit_code" [ "$exit_code" -eq 0 ] echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh000066400000000000000000000066321516554100600325100ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" mkdir work cd work touch ROOT mkdir targets cat > targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": [ "head -c 1 foo/data.txt > out.txt" , "if [ ! -L link ]; then cat link >> out.txt; fi" ] , "deps": ["foo/data.txt", "link"] } } EOF ## Test if non-upwards symlinks work as-is cat > repos.json <&1 grep x "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information # about the root should now be available locally, so we can build without # a serve or remote endpoint with still (logically) fetching absent roots. "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --fetch-absent install -o "${OUT2}" 2>&1 grep x "${OUT2}/out.txt" ## Test if symlinks get resolved cat > repos.json <&1 grep xx "${OUT3}/out.txt" echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-absent-archives.sh000066400000000000000000000124171516554100600306370ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_NON_ABSENT="${TEST_TMPDIR}/local-build-root-non-absent" readonly LBR_FOR_FETCH="${TEST_TMPDIR}/local-build-root-for-fetch" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" readonly OUT_DISTDIR="${TEST_TMPDIR}/out4" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 4.txt > out.txt", "cat 2.txt >> out.txt"] , "deps": ["4.txt", "2.txt"] } } EOF echo cat repos.json echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information # about the root should now be available locally, so we can build without # a serve or remote endpoint with still (logically) fetching absent roots. "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --fetch-absent install -o "${OUT2}" 2>&1 grep 42 "${OUT2}/out.txt" # Now take the same repo, but without the subdir, to ensure we did not # cache a wrong association. cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 src/4.txt > out.txt", "cat src/2.txt >> out.txt"] , "deps": ["src/4.txt", "src/2.txt"] } } EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep 42 "${OUT3}/out.txt" # Now, on a fresh local build root, take the original description # without any absent hint. In this way, we verify that even for # concrete repositories fetching over the just-serve instance works # (as we removed all other ways to get the archive). cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 4.txt > out.txt", "cat 2.txt >> out.txt"] , "deps": ["4.txt", "2.txt"] } } EOF echo cat repos.json echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" ## Finally, verify that the archive itself can also be fetched echo mkdir -p "${OUT_DISTDIR}" "${JUST_MR}" --norc --local-build-root "${LBR_FOR_FETCH}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ fetch -o "${OUT_DISTDIR}" 2>&1 FETCHED_CONTENT=$(git hash-object "${OUT_DISTDIR}"/data.tar) echo echo Fetched content ${FETCHED_CONTENT} [ "${ARCHIVE_CONTENT}" = "${FETCHED_CONTENT}" ] echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh000066400000000000000000000074341516554100600321170ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_NON_ABSENT="${TEST_TMPDIR}/local-build-root-non-absent" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out3" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": [ "tar xf data.tar" , "head -c 1 src/4.txt > out.txt" , "cat src/2.txt >> out.txt" ] , "deps": ["data.tar"] } } EOF echo cat repos.json echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information # about the root should now be available locally, so we can build without # a serve or remote endpoint with still (logically) fetching absent roots. "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --fetch-absent install -o "${OUT2}" 2>&1 grep 42 "${OUT2}/out.txt" # Now, on a fresh local build root, take the original description # without any absent hint. In this way, we verify that even for # concrete repositories fetching over the just-serve instance works # (as we removed all other ways to get the archive). cat > repos.json <&1 grep 42 "${OUT_NON_ABSENT}/out.txt" echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-absent-git-tree.sh000066400000000000000000000064701516554100600305550ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_NON_ABSENT="${TEST_TMPDIR}/local-build-root-non-absent" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 src/4.txt > out.txt", "cat src/2.txt >> out.txt"] , "deps": ["src/4.txt", "src/2.txt"] } } EOF echo cat repos.json echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information # about the root should now be available locally, so we can build without # a serve or remote endpoint with still (logically) fetching absent roots. "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --fetch-absent install -o "${OUT2}" 2>&1 grep 42 "${OUT2}/out.txt" # Now, on a fresh local build root, use the description without any # absent hint. In this way, we verify that even for concrete repositories # fetching over the just-serve instance works (as we removed all other ways # to get the commit). cat > repos.json <&1 grep 42 "${OUT_NON_ABSENT}/out.txt" echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-absent.sh000066400000000000000000000112611516554100600270310ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_NON_ABSENT="${TEST_TMPDIR}/local-build-root-non-absent" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 4.txt > out.txt", "cat 2.txt >> out.txt"] , "deps": ["4.txt", "2.txt"] } } EOF echo cat repos.json echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information # about the root should now be available locally, so we can build without # a serve or remote endpoint with still (logically) fetching absent roots. "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --fetch-absent install -o "${OUT2}" 2>&1 grep 42 "${OUT2}/out.txt" # Now take the same repo, but without the subdir, to ensure we did not # cache a wrong association. cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 src/4.txt > out.txt", "cat src/2.txt >> out.txt"] , "deps": ["src/4.txt", "src/2.txt"] } } EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep 42 "${OUT3}/out.txt" # Now, on a fresh local build root, take the original description # without any absent hint. In this way, we verify that even for # concrete repositories fetching over the just-serve instance works # (as we removed all other ways to get the commit). cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["head -c 1 4.txt > out.txt", "cat 2.txt >> out.txt"] , "deps": ["4.txt", "2.txt"] } } EOF echo cat repos.json echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-gc.sh000066400000000000000000000055241516554100600261530ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly FETCH_TO_DIR="${TEST_TMPDIR}/directory-to-fetch-to" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly INSTALL_DIR="${TEST_TMPDIR}/installation-target" readonly TEST_DATA="The content of the data file in foo" mkdir -p "${DISTDIR}" mkdir -p "${FETCH_TO_DIR}" mkdir -p foo/bar/baz echo {} > foo/TARGETS echo -n "${TEST_DATA}" > foo/bar/baz/data.txt tar cf "${DISTDIR}/foo-1.2.3.tar" foo 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" rm -rf foo # Setup sample repository config touch ROOT cat > repos.json < TARGETS <<'EOF' { "": { "type": "install" , "files": {"out.txt": ["@", "foo", "", "bar/baz/data.txt"]} } } EOF # Call just-mr setup. This will also create a cache entry for the tree # corresponding to that archive "${JUST_MR}" --norc --local-build-root "${LBR}" --distdir "${DISTDIR}" setup 2>&1 # Remove entry from CAS "${JUST}" gc --local-build-root "${LBR}" 2>&1 "${JUST}" gc --local-build-root "${LBR}" 2>&1 # Fetch should still work, if given access to the original file "${JUST_MR}" --norc --local-build-root "${LBR}" --distdir "${DISTDIR}" \ fetch -o "${FETCH_TO_DIR}" 2>&1 newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" # Remove entry again from CAS and clear distdirs "${JUST}" gc --local-build-root "${LBR}" 2>&1 "${JUST}" gc --local-build-root "${LBR}" 2>&1 rm -rf "${DISTDIR}" rm -rf "${FETCH_TO_DIR}" # Setup for building should still be possible "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ install -o "${INSTALL_DIR}" 2>&1 test "$(cat "${INSTALL_DIR}/out.txt")" = "${TEST_DATA}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-remote-git-tree.sh000066400000000000000000000054631516554100600305750ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOCAL_TMPDIR="${TEST_TMPDIR}/local-tmpdir" readonly INSTALL_DIR="${TEST_TMPDIR}/install-dir" # A standard remote-execution server is given by the test infrastructure REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" # Set up folder mkdir -p bin cat > bin/mock-tree < foo/bar/data.txt EOF chmod 755 bin/mock-tree export PATH="${PWD}/bin:${PATH}" # get the tree identifier mkdir -p work ( cd work mock-tree git init git config user.email "nobody@example.org" git config user.name "Nobody" git add -f . git commit -m "Sample output" 2>&1 ) readonly TREE_ID=$(cd work && git log -n 1 --format='%T') rm -rf work echo "dir as tree ${TREE_ID}" # Setup sample repository config touch ROOT cat > repos.json <&1 # Check that tree was fetched "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR}" \ "${TREE_ID}::t" 2>&1 test "$(cat "${INSTALL_DIR}/foo/bar/data.txt")" = "test data" # Clean build root and dirs rm -rf "${LBR}" rm -rf "${LOCAL_TMPDIR}" rm -rf "${INSTALL_DIR}" # Now remove the generating script from the repository description cat > repos.json <&1 # Check that tree was fetched "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR}" \ "${TREE_ID}::t" 2>&1 test "$(cat "${INSTALL_DIR}/foo/bar/data.txt")" = "test data" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch-remote.sh000066400000000000000000000054241516554100600270540ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOCAL_TMPDIR="${TEST_TMPDIR}/local-tmpdir" # A standard remote-execution server is given by the test infrastructure REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" # Setup archive to fetch mkdir -p "${DISTDIR}" mkdir -p foo/bar/baz echo "test data" > foo/bar/baz/data.txt tar cf "${DISTDIR}/foo-1.2.3.tar" foo 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" # Setup sample repository config touch ROOT cat > repos.json <&1 # Remove distdir content rm -rf "${DISTDIR}" mkdir -p "${DISTDIR}" # Fetch to empty distdir while backing up to remote "${JUST_MR}" --norc --local-build-root "${LBR}" \ ${REMOTE_EXECUTION_ARGS} fetch -o "${DISTDIR}" --backup-to-remote 2>&1 # Verify that the correct file is stored in the local tmpdir test -f "${DISTDIR}/foo-1.2.3.tar" newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" # Remove tmpdir content and also the build root rm -rf "${DISTDIR}" mkdir -p "${DISTDIR}" rm -rf "${LBR}" # Now fetch to empty tmpdir using only the remote backup "${JUST_MR}" --norc --local-build-root "${LBR}" \ ${REMOTE_EXECUTION_ARGS} fetch -o "${DISTDIR}" 2>&1 # Verify that the correct file is stored in the local tmpdir test -f "${DISTDIR}/foo-1.2.3.tar" newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/fetch.sh000066400000000000000000000066011516554100600255610ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" mkdir -p foo/bar/baz echo "test data" > foo/bar/baz/data.txt tar cf "${DISTDIR}/foo-1.2.3.tar" foo 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" # Setup sample repository config touch ROOT cat > repos.json <&1 # Remove distdir content rm -rf "${DISTDIR}" mkdir -p "${DISTDIR}" # Ask just-mr to fetch to the empty distdir "${JUST_MR}" --norc --local-build-root "${LBR}" fetch -o "${DISTDIR}" 2>&1 # Verify that the correct file is stored in the distdir test -f "${DISTDIR}/foo-1.2.3.tar" newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" # Verify that fetching accepts distfiles already present "${JUST_MR}" --norc --local-build-root "${LBR}" fetch -o "${DISTDIR}" 2>&1 newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" # Verify we can fetch the same archive if marked absent using --fetch-absent cat > repos.json <&1 # Remove distdir content rm -rf "${DISTDIR}" mkdir -p "${DISTDIR}" # Ask just-mr to fetch to the empty distdir "${JUST_MR}" --norc --local-build-root "${LBR}" --fetch-absent fetch -o "${DISTDIR}" 2>&1 # Verify that the correct file is stored in the distdir test -f "${DISTDIR}/foo-1.2.3.tar" newfoocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has now content ${newfoocontent}" test "${newfoocontent}" = "${foocontent}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/foreign-file.sh000066400000000000000000000052141516554100600270350ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" mkdir -p "${DISTDIR}" echo This-is-the-content > "${DISTDIR}/data.txt" HASH=$(git hash-object "${DISTDIR}/data.txt") # Setup sample repository config touch ROOT cat > repos.json < TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["cat content.txt | tr a-z A-Z > out.txt"] , "deps": ["content.txt"] } } EOF # Build to verify that foreign-file roots work; this will also make just-mr # aware of the file. mkdir -p "${OUT}" "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ -L '["env", "PATH='"${PATH}"'"]' \ --distdir "${DISTDIR}" \ install -o "${OUT}" 2>&1 grep THIS-IS-THE-CONTENT "${OUT}/out.txt" # Remove distdir content rm -rf "${DISTDIR}" mkdir -p "${DISTDIR}" # Ask just-mr to fetch to the empty distdir "${JUST_MR}" --norc --local-build-root "${LBR}" fetch -o "${DISTDIR}" 2>&1 test -f "${DISTDIR}/data.txt" NEW_HASH=$(git hash-object "${DISTDIR}/data.txt") test "${HASH}" = "${NEW_HASH}" # Verify that the root is properly cached, i.e., that we can build again # after cleaning out cache and CAS, and without using the distdir "${JUST}" gc --local-build-root "${LBR}" 2>&1 "${JUST}" gc --local-build-root "${LBR}" 2>&1 rm -f "${OUT}/out.txt" "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ -L '["env", "PATH='"${PATH}"'"]' \ install -o "${OUT}" 2>&1 grep THIS-IS-THE-CONTENT "${OUT}/out.txt" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/gc-drop.sh000066400000000000000000000126261516554100600260270ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly WRKDIR="${PWD}/work" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" readonly ORIG_SRC="${TEST_TMPDIR}/orig" # Generate an archive and make the content known to CAS # ===================================================== mkdir -p "${ORIG_SRC}" cd "${ORIG_SRC}" mkdir -p src/some/deep/directory echo 'some data values' > src/some/deep/directory/data.txt cat > src/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "deps": ["some/deep/directory/data.txt"] , "cmds": ["cat some/deep/directory/data.txt | tr a-z A-Z > out.txt"] } } EOF tar cf source.tar src SOURCE_HASH_A=$("${JUST}" add-to-cas --local-build-root "${LBR}" source.tar) rm -rf source.tar src echo "Upstream hash A is ${SOURCE_HASH_A}" # Generate a second, larger, archive and make the content known to CAS # ==================================================================== mkdir -p "${ORIG_SRC}" cd "${ORIG_SRC}" rm -rf src source.tar mkdir -p src for i in `seq 1 100` do echo "content of file ${i}" > src/data-$i.txt done cat > src/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "deps": [["GLOB", null, "data*.txt"]] , "cmds": ["cat data-*.txt | sort > out.txt"] } } EOF tar cf source.tar src SOURCE_HASH_B=$("${JUST}" add-to-cas --local-build-root "${LBR}" source.tar) rm -rf source.tar src echo "Upstream hash B is ${SOURCE_HASH_B}" # Use both archives # ================= mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.json <&1 # ... sanity check grep VALUES "${OUT}/out.txt" # archive b "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" --main b \ -L '["env", "PATH='"${PATH}"'"]' install -o "${OUT}" 2>&1 # ... sanity check grep 42 "${OUT}/out.txt" # Rotate, and use a again # ======================= "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" --main a \ -L '["env", "PATH='"${PATH}"'"]' install -o "${OUT}" 2>&1 grep VALUES "${OUT}/out.txt" # Verify gc-repo --drop # ===================== # We have the situation that root a is in the young generation, whereas # b is only in the old generation. Therefore, gc-repo --drop-only should # clean up b, which should result in reduced disk usage. # get rid of CAS/cahe "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 # measure disk usage PRE_DROP_DISK=$(du -sb "${LBR}" | cut -f 1) echo "Pre drop, disk usage is ${PRE_DROP_DISK}" # drop "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo --drop-only 2>&1 # measure disk usage POST_DROP_DISK=$(du -sb "${LBR}" | cut -f 1) echo "Post drop, disk usage is ${POST_DROP_DISK}" # post drop the disk usage should be strictly less [ "${POST_DROP_DISK}" -lt "${PRE_DROP_DISK}" ] # Verify that a is still in the youngest generation: even after one more # rotation, we should be able to build a. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" --main a \ -L '["env", "PATH='"${PATH}"'"]' install -o "${OUT}" 2>&1 grep VALUES "${OUT}/out.txt" # Finally demonstrate that the root was not taken from anything but the cache # =========================================================================== "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" --main a \ -L '["env", "PATH='"${PATH}"'"]' build 2>&1 && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/gc-repo.sh000066400000000000000000000156261516554100600260330ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly WORK="${PWD}/work" readonly GIT_REPO="${PWD}/git-repository" readonly BIN="${TEST_TMPDIR}/bin" mkdir -p "${BIN}" readonly DISTDIR="${PWD}/distdir" mkdir -p "${DISTDIR}" readonly ARCHIVE_CONTENT="${TEST_TMPDIR}/archive" readonly SCRATCH_LBR="${TEST_TMPDIR}/throw-away-build-root" readonly SCRATCH_FF="${TEST_TMPDIR}/scratch-space-for-foreign-file" readonly SCRATCH_TREE="${TEST_TMPDIR}/scratch-space-for-git-tree" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" mkdir -p "${OUT}" readonly LOG="${PWD}/log" mkdir -p "${LOG}" # Set up git repo mkdir -p "${GIT_REPO}" cd "${GIT_REPO}" mkdir data echo Hello world > data/text seq 1 10 > data/numbers git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Sample data" 2>&1 git show GIT_COMMIT=$(git log -n 1 --format="%H") GIT_TREE=$(git log -n 1 --format="%T") echo echo "Created commit ${GIT_COMMIT} with tree ${GIT_TREE}" echo ## As we simulate remote fetching, create a mock git binary cat > "${BIN}/mock-git" < description.txt seq 1 20 > data ARCHIVE_TREE=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" .) tar cvf "${DISTDIR}/archive.tar" . ARCHIVE=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" "${DISTDIR}/archive.tar") echo echo "Created archive with content ${ARCHIVE} holding tree ${ARCHIVE_TREE}" echo # Set up a "foreign file" echo Here be dragons > "${DISTDIR}/foreign.txt" FOREIGN=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" "${DISTDIR}/foreign.txt") rm -rf "${SCRATCH_FF}" mkdir -p "${SCRATCH_FF}" cp "${DISTDIR}/foreign.txt" "${SCRATCH_FF}/data.txt" FOREIGN_TREE=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" "${SCRATCH_FF}") echo echo "Foreign file ${FOREIGN} creating tree ${FOREIGN_TREE}" echo # Set up a "git tree" cat > "${BIN}/mock-foreign-vcs" <<'EOF' #!/bin/sh mkdir -p foo/bar/baz echo foo data > foo/data.txt echo bar data > foo/bar/data.txt echo baz data > foo/bar/baz/data.txt EOF chmod 755 "${BIN}/mock-foreign-vcs" rm -rf "${SCRATCH_TREE}" mkdir -p "${SCRATCH_TREE}" (cd "${SCRATCH_TREE}" && "${BIN}/mock-foreign-vcs") FOREIGN_GIT_TREE=$("${JUST}" add-to-cas --local-build-root "${SCRATCH_LBR}" "${SCRATCH_TREE}") echo echo "Git tree ${FOREIGN_GIT_TREE}" echo # Create workspace with just-mr repository configuration mkdir -p "${WORK}" cd "${WORK}" touch ROOT cat > repos.json < "${OUT}/conf-file-name" 2> "${LOG}/log-1" cat "${LOG}/log-1" echo CONFIG="$(cat "${OUT}/conf-file-name")" cat "${CONFIG}" echo ## Verify git repo is set up correctly TREE_FOUND="$(jq -r '.repositories.git.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${GIT_TREE}" ] GIT_LOCATION="$(jq -r '.repositories.git.workspace_root[2]' "${CONFIG}")" ## Verify the archive is set up correctly TREE_FOUND="$(jq -r '.repositories.archive.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${ARCHIVE_TREE}" ] ARCHIVE_LOCATION="$(jq -r '.repositories.archive.workspace_root[2]' "${CONFIG}")" ## Sanity check for foreign file TREE_FOUND="$(jq -r '.repositories.foreign_file.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${FOREIGN_TREE}" ] FOREIGN_LOCATION="$(jq -r '.repositories.foreign_file.workspace_root[2]' "${CONFIG}")" ## Sanity check for git tree TREE_FOUND="$(jq -r '.repositories.git_tree.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${FOREIGN_GIT_TREE}" ] GIT_TREE_LOCATION="$(jq -r '.repositories.git_tree.workspace_root[2]' "${CONFIG}")" # Clean up original repositories rm -rf "${GIT_REPO}" rm -rf "${DISTDIR}" rm -f "${BIN}/mock-foreign-vcs" # Fully clean the non-repo cache "${JUST}" gc --local-build-root "${LBR}" 2>&1 "${JUST}" gc --local-build-root "${LBR}" 2>&1 # Rotate repo cache "${JUST_MR}" --norc --local-build-root "${LBR}" gc-repo 2>&1 ## Verify the mirrored locations are gone [ -e "${GIT_LOCATION}" ] && exit 1 || : [ -e "${ARCHIVE_LOCATION}" ] && exit 1 || : [ -e "${FOREIGN_LOCATION}" ] && exit 1 || : [ -e "${GIT_TREE_LOCATION}" ] && exit 1 || : # Setup repos again "${JUST_MR}" --norc --local-build-root "${LBR}" -L '["env", "PATH='"${PATH}"'"]' \ setup > "${OUT}/conf-file-name" 2> "${LOG}/log-2" cat "${LOG}/log-2" echo CONFIG="$(cat "${OUT}/conf-file-name")" cat "${CONFIG}" echo ## Verify git repo is set up correctly TREE_FOUND="$(jq -r '.repositories.git.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${GIT_TREE}" ] ## Verify the archive is set up correctly TREE_FOUND="$(jq -r '.repositories.archive.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${ARCHIVE_TREE}" ] ## Sanity check for foreign file TREE_FOUND="$(jq -r '.repositories.foreign_file.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${FOREIGN_TREE}" ] ## Sanity check for git tree TREE_FOUND="$(jq -r '.repositories.git_tree.workspace_root[1]' "${CONFIG}")" [ "${TREE_FOUND}" = "${FOREIGN_GIT_TREE}" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/git-env.sh000077500000000000000000000056701516554100600260510ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu JUST_MR="$(pwd)/bin/mr-tool-under-test" JUST="$(pwd)/bin/tool-under-test" WRKDIR="$(pwd)/work" MOCK_TOOLS="$(pwd)/mock-bin" GIT_REPO="${TEST_TMPDIR}/repo" LBR="${TEST_TMPDIR}/local-build-root" LBR2="${TEST_TMPDIR}/local-build-root-2" OUT="${TEST_TMPDIR}/out" # create git repo mkdir -p "${GIT_REPO}" cd "${GIT_REPO}" git init 2>&1 git branch -m stable-1.0 2>&1 echo 'checked-out sources' > sources.txt echo '{}' > TARGETS git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Sample output" 2>&1 git show COMMIT=$(git log -n 1 --format="%H") echo "Created commit ${COMMIT}" # create mock version of git mkdir -p "${MOCK_TOOLS}" MOCK_GIT="${MOCK_TOOLS}/mock-git" cat > "${MOCK_GIT}" <<'EOF' #!/bin/sh if [ "$(cat ${CREDENTIAL_PATH:-/dev/null})" = "sEcReT" ] && [ "$(cat ${LOCAL_CREDENTIAL_PATH:-/dev/null})" = "local-SeCrEt" ] then EOF cat >> "${MOCK_GIT}" <> "${MOCK_GIT}" <<'EOF' else echo 'not enough credentials available' exit 1 fi EOF chmod 755 "${MOCK_GIT}" echo cat "${MOCK_GIT}" echo # Set up client root mkdir -p "${WRKDIR}" cd "${WRKDIR}" mkdir -p etc echo -n sEcReT > etc/pass export CREDENTIAL_PATH="$(pwd)/etc/pass" echo -n local-SeCrEt > etc/local-pass export LOCAL_CREDENTIAL_PATH="$(pwd)/etc/local-pass" mkdir repo cd repo touch ROOT cat > repos.json < local.json <<'EOF' {"extra inherit env": ["LOCAL_CREDENTIAL_PATH"]} EOF # Succesfull build "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ --checkout-locations local.json --git "${MOCK_GIT}" --log-limit 5 \ install -o "${OUT}" '' sources.txt 2>&1 grep checked-out "${OUT}/sources.txt" # Verify the local.json is needed "${JUST_MR}" --norc --git "${MOCK_GIT}" --local-build-root "${LBR2}" setup 2>&1 && exit 1 || : # Verify the environment is needed export CREDENTIAL_PATH=/dev/null "${JUST_MR}" --norc --git "${MOCK_GIT}" --local-build-root "${LBR2}" --checkout-locations local.json setup 2>&1 && exit 1 || : echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/git-tree-env.sh000077500000000000000000000050521516554100600270000ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu JUST_MR="$(pwd)/bin/mr-tool-under-test" JUST="$(pwd)/bin/tool-under-test" LBR="${TEST_TMPDIR}/local-build-root" LBR2="${TEST_TMPDIR}/new-local-build-root" OUT="${TEST_TMPDIR}/out" mkdir -p bin cat > bin/mock-vcs <<'EOF' #!/bin/sh if [ "$(cat ${CREDENTIAL_PATH:-/dev/null})" = "sEcReT" ] && [ "$(cat ${LOCAL_CREDENTIAL_PATH:-/dev/null})" = "local-SeCrEt" ] then echo 'checked-out sources' > sources.txt echo '{}' > TARGETS else echo 'not enough credentials available' fi EOF chmod 755 bin/mock-vcs export PATH="$(pwd)/bin:${PATH}" mkdir -p etc echo -n sEcReT > etc/pass export CREDENTIAL_PATH="$(pwd)/etc/pass" echo -n local-SeCrEt > etc/local-pass export LOCAL_CREDENTIAL_PATH="$(pwd)/etc/local-pass" # Compute tree of our mock checkout mkdir work ( cd work mock-vcs 2>&1 git init 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "Sample output" 2>&1 ) readonly TREE_ID=$(cd work && git log -n 1 --format="%T") rm -rf work echo "Tree of checkout is ${TREE_ID}." # Setup repo config mkdir repo cd repo touch ROOT cat > repos.json < local.json <<'EOF' {"extra inherit env": ["LOCAL_CREDENTIAL_PATH"]} EOF # Succesfull build "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ --checkout-locations local.json \ install -o "${OUT}" '' sources.txt 2>&1 grep checked-out "${OUT}/sources.txt" # Verify the local.json is needed "${JUST_MR}" --norc --local-build-root "${LBR2}" setup 2>&1 && exit 1 || : # Verify the environment is needed export CREDENTIAL_PATH=/dev/null "${JUST_MR}" --norc --local-build-root "${LBR2}" \ --checkout-locations local.json \ setup 2>&1 && exit 1 || : echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/ignore-special.sh000066400000000000000000000057741516554100600274030ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that special entries, be it symlinks or git submodules, are # properly ignored during setup of git roots when the "special":"ignore" pragma # is provided. ## set -eu env readonly ROOT="${PWD}" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/build-output" readonly REPO_DIRS="${TEST_TMPDIR}/repos" readonly WRKDIR="${PWD}/work" # Create a Git repo 'foo' mkdir -p "${REPO_DIRS}/foo" cd "${REPO_DIRS}/foo" git init git checkout --orphan foomaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' touch data git add . -f git commit -m 'Init foo' 2>&1 # Create a Git repo 'foo' with a submodule mkdir -p "${REPO_DIRS}/bar" cd "${REPO_DIRS}/bar" git init git checkout --orphan barmaster git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git -c protocol.file.allow=always submodule add "${REPO_DIRS}/foo/.git" # submodule foo ln -s ../nonexistent a_link # a symlink to be ignored echo '{"":{"type":"install","deps":[["TREE", null, "."]]}}' >TARGETS git add . -f git commit -m 'Init bar' 2>&1 # Main repo depending on repo 'bar' mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT echo '{"":{"type":"install","deps":[["@", "bar", "", ""]]}}' >TARGETS echo echo Check git root fails without pragma:ignore echo cat > repos.json << EOF { "main": "" , "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"bar": "bar"} } , "bar": { "repository": { "type": "file" , "path": "${REPO_DIRS}/bar" , "pragma": {"to_git": true} } } } } EOF "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" build 2>&1 \ && echo "this should fail" && exit 1 echo echo "failed as expected" cat > repos.json << EOF { "main": "" , "repositories": { "": { "repository": {"type": "file", "path": "."} , "bindings": {"bar": "bar"} } , "bar": { "repository": { "type": "file" , "path": "${REPO_DIRS}/bar" , "pragma": {"to_git": true, "special":"ignore"} } } } } EOF echo echo Check git root with pragma:ignore succeeds echo "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" install -o "${OUT}" 2>&1 [ ! -e "${OUT}/a_link" ] # symlink should not be there [ ! -e "${OUT}/foo" ] # submodule should not be there echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/install-roots-symlinks.sh000066400000000000000000000241171516554100600311530ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test extends install-roots-basic with 'special' pragma use-cases ## set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly INSTALL_DIR_RESOLVED="${TEST_TMPDIR}/installation-target-resolved" readonly INSTALL_DIR_UNRESOLVED="${TEST_TMPDIR}/installation-target-unresolved" readonly INSTALL_DIR_SPECIAL_IGNORE="${TEST_TMPDIR}/installation-target-special-ignore" readonly INSTALL_DIR_SPECIAL_PARTIAL="${TEST_TMPDIR}/installation-target-special-partial" readonly INSTALL_DIR_SPECIAL_COMPLETE="${TEST_TMPDIR}/installation-target-special-complete" readonly TEST_DATA="The content of the data file in foo" readonly TEST_DIRS="bar/baz" readonly DATA_PATH="bar/baz/data.txt" readonly NON_UPWARDS_LINK_PATH="bar/baz/nonupwards" readonly NON_UPWARDS_LINK_TARGET="data.txt" readonly UPWARDS_LINK_PATH="bar/upwards" readonly UPWARDS_LINK_TARGET="../bar/baz" readonly INDIRECT_LINK_PATH="bar/indirect" readonly INDIRECT_LINK_TARGET="upwards/data.txt" readonly CYCLE_LINK_1_PATH="bar/cycle_1" readonly CYCLE_LINK_1_TARGET="cycle_2" readonly CYCLE_LINK_2_PATH="bar/cycle_2" readonly CYCLE_LINK_2_TARGET="cycle_1" mkdir -p "${DISTDIR}" mkdir -p log LOGDIR="$(realpath log)" LOG_ARCHIVE_REPO="${LOGDIR}/archive.txt" LOG_FILE_REPO="${LOGDIR}/file.txt" ### # Set up sample archives ## ROOT=$(pwd) mkdir -p "foo/${TEST_DIRS}" echo {} > foo/TARGETS echo -n "${TEST_DATA}" > "foo/${DATA_PATH}" # add resolvable non-upwards symlink ln -s "${NON_UPWARDS_LINK_TARGET}" "foo/${NON_UPWARDS_LINK_PATH}" tar cf "${DISTDIR}/foo-1.2.3_v1.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 foocontent_v1=$(git hash-object "${DISTDIR}/foo-1.2.3_v1.tar") echo "Foo archive v1 has content ${foocontent_v1}" # get Git-tree of foo subdir with only non-upwards symlinks cd foo ( git init \ && git config user.email "nobody@example.org" \ && git config user.name "Nobody" \ && git add . \ && git commit -m "test" --date="1970-01-01T00:00Z" ) UNRESOLVED_TREE=$(git log -n 1 --format='%T') echo "Foo archive v1 has unresolved tree ${UNRESOLVED_TREE}" rm -rf .git cd .. # now add also a resolvable upwards symlink ln -s "${UPWARDS_LINK_TARGET}" "foo/${UPWARDS_LINK_PATH}" # ...and a resolvable non-upwards symlink pointing to it ln -s "${INDIRECT_LINK_TARGET}" "foo/${INDIRECT_LINK_PATH}" tar cf "${DISTDIR}/foo-1.2.3_v2.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 foocontent_v2=$(git hash-object "${DISTDIR}/foo-1.2.3_v2.tar") echo "Foo archive v2 has content ${foocontent_v2}" # lastly, check that symlink cycles will fail to resolve ln -s "${CYCLE_LINK_1_TARGET}" "foo/${CYCLE_LINK_1_PATH}" ln -s "${CYCLE_LINK_2_TARGET}" "foo/${CYCLE_LINK_2_PATH}" tar cf "${DISTDIR}/foo-1.2.3_cycle.tar" foo \ --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 foocontent_cycle=$(git hash-object "${DISTDIR}/foo-1.2.3_cycle.tar") echo "Foo archive with cycle has content ${foocontent_cycle}" ### # Setup sample repository config ## touch ROOT cat > repos.json <&1 test "$(cat "${INSTALL_DIR_RESOLVED}/${DATA_PATH}")" = "${TEST_DATA}" # non-upwards symlink is resolved test "$(cat "${INSTALL_DIR_RESOLVED}/${NON_UPWARDS_LINK_PATH}")" = "${TEST_DATA}" # A resolved tree should add to CAS also its unresolved version; in our case, # the unresolved tree has only non-upwards symlinks, so it can be installed "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_UNRESOLVED}" \ "${UNRESOLVED_TREE}::t" 2>&1 test "$(cat "${INSTALL_DIR_UNRESOLVED}/${DATA_PATH}")" = "${TEST_DATA}" test "$(readlink "${INSTALL_DIR_UNRESOLVED}/${NON_UPWARDS_LINK_PATH}")" = "${NON_UPWARDS_LINK_TARGET}" ### # Test archives with confined symlinks (upwards and non-upwards) ## echo === root with ignored special entries === # Read tree from repository configuration TREE=$(jq -r '.repositories.foo_v2_ignore_special.workspace_root[1]' "${CONF}") echo Tree is "${TREE}" # As the tree is known to just (in the git CAS), we should be able to install # it with install-cas "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_IGNORE}" \ "${TREE}::t" 2>&1 test "$(cat "${INSTALL_DIR_SPECIAL_IGNORE}/${DATA_PATH}")" = "${TEST_DATA}" [ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${NON_UPWARDS_LINK_PATH}" ] # symlink should be missing [ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${UPWARDS_LINK_PATH}" ] # symlink should be missing [ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${INDIRECT_LINK_PATH}" ] # symlink should be missing echo === root with partially resolved symlinks === # Read tree from repository configuration TREE=$(jq -r '.repositories.foo_v2_resolve_partially.workspace_root[1]' "${CONF}") echo Tree is "${TREE}" # As the tree is known to just (in the git CAS), we should be able to install # it with install-cas "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_PARTIAL}" \ "${TREE}::t" 2>&1 test "$(cat "${INSTALL_DIR_SPECIAL_PARTIAL}/${DATA_PATH}")" = "${TEST_DATA}" # non-upwards link remains as-is test "$(readlink "${INSTALL_DIR_SPECIAL_PARTIAL}/${NON_UPWARDS_LINK_PATH}")" = "${NON_UPWARDS_LINK_TARGET}" # upwards link is resolved, in this case to a tree [ -d "${INSTALL_DIR_SPECIAL_PARTIAL}/${UPWARDS_LINK_PATH}" ] test "$(cat "${INSTALL_DIR_SPECIAL_PARTIAL}/${UPWARDS_LINK_PATH}/data.txt")" = "${TEST_DATA}" # indirect link is non-upwards, so it should remain as-is test "$(readlink "${INSTALL_DIR_SPECIAL_PARTIAL}/${INDIRECT_LINK_PATH}")" = "${INDIRECT_LINK_TARGET}" echo === root with fully resolved symlinks === # Read tree from repository configuration TREE=$(jq -r '.repositories.foo_v2_resolve_completely.workspace_root[1]' "${CONF}") echo Tree is "${TREE}" # As the tree is known to just (in the git CAS), we should be able to install # it with install-cas "${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_COMPLETE}" \ "${TREE}::t" 2>&1 test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${DATA_PATH}")" = "${TEST_DATA}" # all links get resolved test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${NON_UPWARDS_LINK_PATH}")" = "${TEST_DATA}" [ -d "${INSTALL_DIR_SPECIAL_COMPLETE}/${UPWARDS_LINK_PATH}" ] test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${UPWARDS_LINK_PATH}/data.txt")" = "${TEST_DATA}" test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${INDIRECT_LINK_PATH}")" = "${TEST_DATA}" echo === symlink cycle detection === "${JUST_MR}" --norc --local-build-root "${LBR}" --distdir "${DISTDIR}" \ -f "${LOG_ARCHIVE_REPO}" setup foo_cycle_archive \ && echo "this should fail" >&2 && exit 1 echo "failed as expected" # check for cycle in log file grep "${CYCLE_LINK_2_PATH}" "${LOG_ARCHIVE_REPO}" grep "${CYCLE_LINK_1_PATH}" "${LOG_ARCHIVE_REPO}" "${JUST_MR}" --norc --local-build-root "${LBR}" --distdir "${DISTDIR}" \ -f "${LOG_FILE_REPO}" setup foo_cycle_file \ && echo "this should fail" >&2 && exit 1 echo "failed as expected" # check for cycle in log file grep "${CYCLE_LINK_1_PATH}" "${LOG_FILE_REPO}" grep "${CYCLE_LINK_2_PATH}" "${LOG_FILE_REPO}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/install-roots.sh000066400000000000000000000044611516554100600273040ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly INSTALL_DIR="${TEST_TMPDIR}/installation-target" readonly TEST_DATA="The content of the data file in foo" readonly TEST_PATH="bar/baz" readonly LINK_TARGET="dummy" mkdir -p "${DISTDIR}" # Set up sample archive mkdir -p "foo/${TEST_PATH}" echo {} > foo/TARGETS echo -n "${TEST_DATA}" > "foo/${TEST_PATH}/data.txt" ln -s "${LINK_TARGET}" "foo/${TEST_PATH}/link" tar cf "${DISTDIR}/foo-1.2.3.tar" foo 2>&1 foocontent=$(git hash-object "${DISTDIR}/foo-1.2.3.tar") echo "Foo archive has content ${foocontent}" rm -rf foo # Setup sample repository config touch ROOT cat > repos.json <&1 test "$(cat "${INSTALL_DIR}/${TEST_PATH}/data.txt")" = "${TEST_DATA}" test "$(readlink "${INSTALL_DIR}/${TEST_PATH}/link")" = "${LINK_TARGET}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/invocation-log.sh000066400000000000000000000070151516554100600274200ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${PWD}/log" readonly ETC_DIR="${PWD}/etc" readonly WRK_DIR="${PWD}/work" readonly REPORTED_CONFIG="${PWD}/out/just-repo-config.json" readonly OUT_GRAPH="${TEST_TMPDIR}/cmdline-option-graph.json" # Set up an rc file, requesting invocation logging mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" < repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > rc.json <<'EOF' {"invocation log": {"project id": "invocation-log-test"}} EOF cat > TARGETS <<'EOF' { "upper": { "type": "generic" , "deps": ["data.txt"] , "outs": ["upper.txt"] , "cmds": ["cat data.txt | tr a-z A-Z > upper.txt"] } } EOF echo blablabla > data.txt # Call analyse via just-mr "${JUST_MR}" --rc "${RC}" analyse --dump-graph "${OUT_GRAPH}" upper 2>&1 # As this is the first invocation, we can find the invocation-log dir by a glob INVOCATION_DIR="$(ls -d "${LOG_DIR}"/invocation-log-test/*)" # ... this should create a metadata file. METADATA_FILE="${INVOCATION_DIR}/meta.json" echo "Meta data file ${METADATA_FILE}" cat "${METADATA_FILE}" echo # Sanity check: the local build root must occur in the command line as # it was specified in the rc file. [ $(jq '.cmdline | contains(["'"${LBR}"'"])' "${METADATA_FILE}") = true ] # Install the referenced configuration "${JUST_MR}" --rc "${RC}" install-cas -o "${REPORTED_CONFIG}" \ $(jq -r '.configuration' "${METADATA_FILE}") 2>&1 echo cat "${REPORTED_CONFIG}" echo # ... and verify that the file repository contains the correct path [ "$(jq -r '.repositories.""."workspace_root"[1]' "${REPORTED_CONFIG}")" = "${WRK_DIR}" ] # Verify the presence of graph file GRAPH_FILE="${INVOCATION_DIR}/graph.json" [ -f "${GRAPH_FILE}" ] echo "Graph file ${GRAPH_FILE}" cat "${GRAPH_FILE}" # ... and do some basic sanity check [ $(jq '.actions | keys | length' "${GRAPH_FILE}") -eq 1 ] ACTION_KEY="$(jq '.actions | keys | .[0]' "${GRAPH_FILE}")" [ "$(jq -r '.actions.'"${ACTION_KEY}"'.output[0]' "${GRAPH_FILE}")" = "upper.txt" ] # Also verify that the passed graph option is honored as well [ -f "${OUT_GRAPH}" ] cmp "${GRAPH_FILE}" "${OUT_GRAPH}" # Verify the profile file PROFILE_FILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE_FILE}" [ $(jq '."exit code"' "${PROFILE_FILE}") -eq 0 ] [ $(jq -r '."target"[3]' "${PROFILE_FILE}") = "upper" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/just-mr-mirrors.test.sh000066400000000000000000000236711516554100600305500ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu # cleanup of http.server; pass server_pid as arg server_cleanup() { echo "Shut down HTTP server(s)" while [ -n "${1:-}" ]; do # send SIGTERM kill ${1} & res=$! wait ${res} echo "done ${1}" shift done } readonly ROOT=`pwd` readonly WRKDIR="${TEST_TMPDIR}/wrkdir" readonly JUST_MR_CPP="${ROOT}/bin/mr-tool-under-test" # set paths readonly SERVER_ROOT="${TEST_TMPDIR}/server-root" readonly GIT_ROOT="${TEST_TMPDIR}/git-root" readonly TEST_ROOT="${TEST_TMPDIR}/test-root" # move to where server will be posted mkdir -p "${SERVER_ROOT}" cd "${SERVER_ROOT}" echo "Setup archive repos" # create the archives in current dir ${ROOT}/src/create_test_archives # store zip repo info readonly ZIP_REPO_CONTENT=$(git hash-object zip_repo.zip) readonly ZIP_REPO_SHA256=$(sha256sum zip_repo.zip | awk '{print $1}') readonly ZIP_REPO_SHA512=$(sha512sum zip_repo.zip | awk '{print $1}') # store tar.gz repo info readonly TGZ_REPO_CONTENT=$(git hash-object tgz_repo.tar.gz) readonly TGZ_REPO_SHA256=$(sha256sum tgz_repo.tar.gz | awk '{print $1}') readonly TGZ_REPO_SHA512=$(sha512sum tgz_repo.tar.gz | awk '{print $1}') echo "Set up local git repo" # NOTE: Libgit2 has no support for Git bundles yet mkdir -p "${GIT_ROOT}" mkdir -p bin cat > bin/git_dir_setup.sh < foo.txt echo bar > foo/bar.txt echo baz > foo/bar/baz.txt ln -s dummy foo/link EOF # set up git repo ( cd "${GIT_ROOT}" \ && sh "${SERVER_ROOT}/bin/git_dir_setup.sh" \ && git init \ && git checkout -q -b test \ && git config user.email "nobody@example.org" \ && git config user.name "Nobody" \ && git add . \ && git commit -m "test" --date="1970-01-01T00:00Z" ) readonly GIT_REPO_COMMIT="$(cd "${GIT_ROOT}" && git rev-parse HEAD)" echo "Publish remote repos to HTTP server" # start Python server as remote repos location port_file="$(mktemp -t port_XXXXXX -p "${TEST_TMPDIR}")" python3 -u "${ROOT}/utils/run_test_server.py" "${port_file}" & server_pid=$! # set up cleanup of http server trap "server_cleanup ${server_pid}" INT TERM EXIT # wait for the server to be available tries=0 while [ -z "$(cat "${port_file}")" ] && [ $tries -lt 10 ] do tries=$((${tries}+1)) sleep 1s done # get port number as variable port_num="$(cat ${port_file})" if [ -z "${port_num}" ]; then exit 1 fi echo "Create local build env" # the build root readonly BUILDROOT=${WRKDIR}/.cache/just mkdir -p ${BUILDROOT} # a distdir readonly DISTFILES=${WRKDIR}/.distfiles mkdir -p ${DISTFILES} readonly DISTDIR_ARGS="--distdir ${DISTFILES}" ### Using "mirrors" config field echo "Create repos.json" mkdir -p "${TEST_ROOT}" cd "${TEST_ROOT}" cat > test-repos.json < test-repos.json < just-local.json < test-repos.json < just-local.json < "${FILE_ROOT}/test_dir1/test_file" mkdir -p "${FILE_ROOT}/test_dir2/test_dir3" echo test > "${FILE_ROOT}/test_dir2/test_dir3/test_file" # add resolvable non-upwards symlink ln -s test_file "${FILE_ROOT}/test_dir1/nonupwards" # add resolvable upwards symlink ln -s ../test_dir3/test_file "${FILE_ROOT}/test_dir2/test_dir3/upwards" echo "Set up local git repo" # NOTE: Libgit2 has no support for Git bundles yet mkdir -p "${GIT_ROOT}" mkdir -p bin cat > bin/git_dir_setup.sh < foo.txt echo bar > foo/bar.txt echo baz > foo/bar/baz.txt ln -s dummy foo/link EOF # set up git repo ( cd "${GIT_ROOT}" \ && sh "${SERVER_ROOT}/bin/git_dir_setup.sh" \ && git init \ && git checkout -q -b test \ && git config user.email "nobody@example.org" \ && git config user.name "Nobody" \ && git add . \ && git commit -m "test" --date="1970-01-01T00:00Z" ) readonly GIT_REPO_COMMIT="$(cd "${GIT_ROOT}" && git rev-parse HEAD)" echo "Set up git tree repo" # get tree id; only the content of the directory structure matters readonly GIT_TREE_ID="$(cd "${GIT_ROOT}" && git ls-tree "${GIT_REPO_COMMIT}" foo/bar | awk '{print $3}')" echo "Publish remote repos to HTTP server" # start Python server as remote repos location port_file="$(mktemp -t port_XXXXXX -p "${TEST_TMPDIR}")" python3 -u "${ROOT}/utils/run_test_server.py" "${port_file}" & server_pid=$! # set up cleanup of http server trap "server_cleanup ${server_pid}" INT TERM EXIT # wait for the server to be available tries=0 while [ -z "$(cat "${port_file}")" ] && [ $tries -lt 10 ] do tries=$((${tries}+1)) sleep 1s done # get port number as variable port_num="$(cat ${port_file})" if [ -z "${port_num}" ]; then exit 1 fi echo "Create local build env" # the build root readonly BUILDROOT=${WRKDIR}/.cache/just mkdir -p ${BUILDROOT} # a distdir readonly DISTFILES=${WRKDIR}/.distfiles mkdir -p ${DISTFILES} readonly DISTDIR_ARGS="--distdir ${DISTFILES}" echo "Create repos.json" mkdir -p "${TEST_ROOT}" cd "${TEST_ROOT}" cat > test-repos.json < $1 OK" } test_alone zip_repo test_alone zip_repo_resolved test_alone tgz_repo test_alone git_repo test_alone git_repo_ignore_special test_alone git_tree_repo test_alone git_tree_repo_ignore_special test_alone file_repo1 test_alone file_repo2_ignore_special test_alone file_repo2_resolve_partially test_alone file_repo2_resolve_completely test_alone distdir_repo ### Check parallel run echo "Set up parallel run" test_all() { CONFIG_CPP=$("${JUST_MR_CPP}" -C test-repos.json --norc \ --local-build-root "${BUILDROOT}" \ -L '["env", "PATH='"${PATH}"'"]' \ ${DISTDIR_ARGS} -j 32 setup --all) if [ ! -s "${CONFIG_CPP}" ]; then exit 1 fi } echo "Run 8 test cases in parallel" error=false test_all & res1=$! test_all & res2=$! test_all & res3=$! test_all & res4=$! test_all & res5=$! test_all & res6=$! test_all & res7=$! test_all & res8=$! wait ${res1} if [ $? -ne 0 ]; then error=true fi wait ${res2} if [ $? -ne 0 ]; then error=true fi wait ${res3} if [ $? -ne 0 ]; then error=true fi wait ${res4} if [ $? -ne 0 ]; then error=true fi wait ${res5} if [ $? -ne 0 ]; then error=true fi wait ${res6} if [ $? -ne 0 ]; then error=true fi wait ${res7} if [ $? -ne 0 ]; then error=true fi wait ${res8} if [ $? -ne 0 ]; then error=true fi # check test status if [ $error = true ]; then exit 1 fi just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/repeated-gc.sh000077500000000000000000000104771516554100600266610ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly WRKDIR="${PWD}/work" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" readonly ORIG_SRC="${TEST_TMPDIR}/orig" # Generate an archive and make the content known to CAS # ===================================================== mkdir -p "${ORIG_SRC}" cd "${ORIG_SRC}" mkdir -p src/some/deep/directory echo 'some data values' > src/some/deep/directory/data.txt cat > src/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt"] , "deps": ["some/deep/directory/data.txt"] , "cmds": ["cat some/deep/directory/data.txt | tr a-z A-Z > out.txt"] } } EOF tar cvf source.tar src SOURCE_HASH=$("${JUST}" add-to-cas --local-build-root "${LBR}" source.tar) rm -rf source.tar src echo "Upstream hash is ${SOURCE_HASH}" # Use that archive # ================ mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.json <&1 # sanity check grep VALUES "${OUT}/out.txt" # Clean up build root: CAS/cache fully, rotate repo cache # ======================================================= # Before rotation the git root should still exist [ -e "${GIT_ROOT}" ] "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 # After gc rotation, the original git root should no longer exist [ -e "${GIT_ROOT}" ] && exit 1 || : "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 # Building should nevertheless succeed, due to the old repo generation # ===================================================================== "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' install -o "${OUT}" 2>&1 # sanity check grep VALUES "${OUT}/out.txt" rm -f "${OUT}/out.txt" # Repeat the whole procedure several times # ======================================== for _ in `seq 1 5` do CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" setup) GIT_ROOT=$(jq -r '.repositories.""."workspace_root" | .[2]' "${CONF}") echo "Git root is ${GIT_ROOT}" [ -e "${GIT_ROOT}" ] "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 [ -e "${GIT_ROOT}" ] && exit 1 || : "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc 2>&1 "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' install -o "${OUT}" 2>&1 grep VALUES "${OUT}/out.txt" rm -f "${OUT}/out.txt" done # Finally demonstrate that the root was not taken from anything but the cache # =========================================================================== "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" gc-repo 2>&1 # after full rotation of the repository, the root should be lost "${JUST_MR}" --norc --local-build-root "${LBR}" -f "${OUT}/log" \ setup 2>&1 && exit 1 || : # sanity check: the error message should mention the fetch host grep nonexistent.upstream.example.com "${OUT}/log" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/reporting-verbosity.sh000066400000000000000000000041621516554100600305250ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" readonly LOG="${TEST_TMPDIR}/log" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["archive_id"] , "cmds": ["git hash-object data.tar > archive_id"] , "deps": ["data.tar"] } } EOF echo cat repos.json echo mkdir -p "${LOG}" "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} \ -f "${LOG}/log" \ --distdir ../src \ install -o "${OUT}" 2>&1 echo cat "${OUT}/archive_id" [ $(cat "${OUT}/archive_id") = "${ARCHIVE_CONTENT}" ] echo # As the build succeeded, there should not be any error reported cat "${LOG}/log" echo grep ERROR "${LOG}/log" && exit 1 || : echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/stay-local.sh000066400000000000000000000065431516554100600265450ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR2="${TEST_TMPDIR}/local-build-root-2" readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out-2" readonly LOG="${TEST_TMPDIR}/log" readonly EMPTY="${TEST_TMPDIR}/empty-directory" readonly SERVER="${PWD}/utils/null-server" readonly SERVER_STATE="${TEST_TMPDIR}/server" ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" mkdir -p "${SERVER_STATE}" "${SERVER}" "${SERVER_STATE}/port" "${SERVER_STATE}/access" & server_pid=$! trap "kill $server_pid" INT TERM EXIT while [ '!' -f "${SERVER_STATE}/port" ] do sleep 1s done # get port number as variable port="$(cat "${SERVER_STATE}/port")" echo "Server up and listening at port ${port}" mkdir work cd work touch ROOT cat > repos.json < targets/TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["archive_id"] , "cmds": ["git hash-object data.tar > archive_id"] , "deps": ["data.tar"] } } EOF echo cat repos.json echo mkdir -p "${LOG}" "${JUST_MR}" --norc --local-build-root "${LBR}" \ -r "127.0.0.1:${port}" \ --log-limit 5 -f "${LOG}/log" \ --distdir ../src \ setup > conf.json echo cat "${LOG}/log" echo cat conf.json echo cat $(cat conf.json) echo # As a distdir (local directory!) was provided with all needed files, # no attempt should be made to contact the remote-execution endpoint echo [ -f "${SERVER_STATE}/access" ] && cat "${SERVER_STATE}/access" && exit 1 || : # The obtained configuraiton should be suitable for building, also remotely "${JUST}" install -C "$(cat conf.json)" -o "${OUT}" \ --local-build-root "${LBR}" \ -r "${REMOTE_EXECUTION_ADDRESS}" 2>&1 echo cat "${OUT}/archive_id" [ $(cat "${OUT}/archive_id") = "${ARCHIVE_CONTENT}" ] echo # As the archive in question was input to an action, setup via the # remote-execution endpoint should work now, even if the provided # distdir is empty mkdir -p "${EMPTY}" "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR2}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ --distdir ${EMPTY} \ install -o "${OUT2}" 2>&1 cat "${OUT2}/archive_id" [ $(cat "${OUT2}/archive_id") = "${ARCHIVE_CONTENT}" ] echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/just-mr/verbosity.sh000066400000000000000000000051021516554100600265110ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly COMPUTE_TREE_DIR="${TEST_TMPDIR}/generate-the-tree-id" readonly LBR="${TEST_TMPDIR}/local-build-root" # Set up a tree generating tool, and the corresponding tree readonly TOOL_MSG_A="UsefulTextOnStout" readonly TOOL_MSG_B="PotentialErrorOnStderr" mkdir -p "${TOOLS_DIR}" readonly TOOL="${TOOLS_DIR}/generate-tree.sh" cat > "${TOOL}" <&2 echo {} > TARGETS if [ -n "\${SHOULD_FAIL:-}" ] then exit 0 fi echo data > foo.txt EOF chmod 755 "${TOOL}" mkdir -p "${COMPUTE_TREE_DIR}" ( cd "${COMPUTE_TREE_DIR}" git init git config user.email "nobody@example.org" git config user.name "Nobody" "${TOOL}" git add . git commit -m "Sample output" ) readonly TREE_ID=$(cd "${COMPUTE_TREE_DIR}" && git log -n 1 --format="%T") rm -rf "${COMPUTE_TREE_DIR}" # Set up workspace touch ROOT cat > repos.json <log && exit 1 || : cat log grep -q -F "${TOOL_MSG_A}" log grep -q -F "${TOOL_MSG_B}" log grep -q -F "SHOULD_FAIL=YES" log grep -q -F "${TOOL}" log # On success, everything should be quiet (despite the command # potentially producing output) echo echo testing setup success echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "SHOULD_FAIL="]' setup 2>log) cat log echo "${CONF}" cat "${CONF}" echo grep -F "${TOOL_MSG_A}" log && exit 1 || : grep -F "${TOOL_MSG_B}" log && exit 1 || : grep -F "${TOOL}" log && exit 1 || : # sanity check: the generated configuration contains the requested tree grep -q -F "${TREE_ID}" "${CONF}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/000077500000000000000000000000001516554100600241705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/TARGETS000066400000000000000000000022621516554100600252260ustar00rootroot00000000000000{ "basic": { "type": ["@", "rules", "shell/test", "script"] , "name": ["basic"] , "test": ["basic.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] } , "failing build": { "type": ["@", "rules", "shell/test", "script"] , "name": ["failing"] , "test": ["failing.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] } , "time": { "type": ["@", "rules", "shell/test", "script"] , "name": ["time"] , "test": ["time.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] } , "time, remote": { "type": ["end-to-end", "with remote"] , "name": ["time-remote"] , "test": ["time.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] } , "analysis": { "type": ["@", "rules", "shell/test", "script"] , "name": ["analysis"] , "test": ["analysis.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep-dirs": ["log"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["profile"] , "deps": ["analysis", "basic", "failing build", "time", "time, remote"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/analysis.sh000066400000000000000000000070521516554100600263530ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${PWD}/log" readonly ETC_DIR="${PWD}/etc" readonly WRK_DIR="${PWD}/work" readonly OUT="${TEST_TMPDIR}/out" # Set up an rc file, requesting invocation logging mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" < repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "flex file": { "type": "generic" , "arguments_config": ["FLEX_FILE"] , "outs": [{"type": "var", "name": "FLEX_FILE"}] , "cmds": [ { "type": "join" , "$1": ["echo content > ", {"type": "var", "name": "FLEX_FILE"}] } ] } , "flex dir": { "type": "generic" , "arguments_config": ["FLEX_DIR"] , "out_dirs": [{"type": "var", "name": "FLEX_DIR"}] , "cmds": [ { "type": "join_cmd" , "$1": ["mkdir", "-p", {"type": "var", "name": "FLEX_DIR"}] } , { "type": "join" , "$1": ["echo content > ", {"type": "var", "name": "FLEX_DIR"}, "/foo.txt"] } , { "type": "join" , "$1": ["echo content > ", {"type": "var", "name": "FLEX_DIR"}, "/bar.txt"] } ] } , "": { "type": "generic" , "outs": ["ls.txt"] , "cmds": ["touch ls.txt", "find . -type f | sort > ls.txt"] , "deps": ["flex file", "flex dir"] } } EOF cat > rc.json <<'EOF' {"invocation log": {"project id": "good-build"}} EOF "${JUST_MR}" --rc "${RC}" build -p \ -D '{"FLEX_DIR": "some/dir", "FLEX_FILE": "path/to/file.txt"}' 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/good-build/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # exit code of just should be reported as 0 [ $(jq '."exit code"' "${PROFILE}") -eq 0 ] # There should be no analysis errors [ $(jq '."analysis errors" | length' "${PROFILE}") -eq 0 ] cat > rc.json <<'EOF' {"invocation log": {"project id": "analysis"}} EOF "${JUST_MR}" --rc "${RC}" build -p \ -D '{"FLEX_DIR": "path/to/dir", "FLEX_FILE": "path/to/dir/file.txt"}' 2>&1 \ && exit 1 || : INVOCATION_DIR="$(ls -d "${LOG_DIR}"/analysis/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # exit code of just should be reported as 8 (analysis) [ $(jq '."exit code"' "${PROFILE}") -eq 8 ] # we expect one entry in analysis errors [ $(jq '."analysis errors" | length' "${PROFILE}") -eq 1 ] # ... and the error message should describe the problem jq -r '."analysis errors"[0]' "${PROFILE}" | grep 'stag.*conflict.*path/to/dir' echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/basic.sh000066400000000000000000000112741516554100600256120ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${PWD}/log" readonly ETC_DIR="${PWD}/etc" readonly WRK_DIR="${PWD}/work" readonly OUT="${TEST_TMPDIR}/out" # Set up an rc file, requesting invocation logging mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" < repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "upper": { "type": "generic" , "deps": ["data.txt"] , "outs": ["upper.txt"] , "cmds": [ "cat data.txt | tr a-z A-Z > upper.txt" , "echo StdOuT" , "echo StdErR 1>&2" ] } } EOF echo blablabla > data.txt # Call analyse via just-mr; abuse the project id to distingush the runs cat > rc.json <<'EOF' {"invocation log": {"project id": "first-run"}} EOF MY_CONTEXT=TeStCoNtExT "${JUST_MR}" --rc "${RC}" build upper 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/first-run/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" echo [ $(jq '."exit code"' "${PROFILE}") -eq 0 ] [ $(jq -r '."subcommand"' "${PROFILE}") = "build" ] [ $(jq -r '."target" | .[3]' "${PROFILE}") = "upper" ] [ $(jq '.actions | .[] | .cached' "${PROFILE}") = "false" ] OUT_ARTIFACT=$(jq -r '.actions | .[] | .artifacts."upper.txt"' "${PROFILE}") "${JUST_MR}" --rc "${RC}" install-cas -o "${OUT}/upper.txt" "${OUT_ARTIFACT}" 2>&1 grep BLABLABLA "${OUT}/upper.txt" STDOUT=$(jq -r '.actions | .[] | .stdout' "${PROFILE}") STDERR=$(jq -r '.actions | .[] | .stderr' "${PROFILE}") "${JUST_MR}" --rc "${RC}" install-cas -o "${OUT}/stdout" "${STDOUT}" 2>&1 "${JUST_MR}" --rc "${RC}" install-cas -o "${OUT}/stderr" "${STDERR}" 2>&1 grep StdOuT "${OUT}/stdout" grep StdErR "${OUT}/stderr" ARTIFACTS="${INVOCATION_DIR}/artifacts.json" cat "${ARTIFACTS}" echo [ $(jq -r '."upper.txt".id' "${ARTIFACTS}") = "${OUT_ARTIFACT}" ] META="${INVOCATION_DIR}/meta.json" cat "${META}" echo [ $(jq -r '.cmdline | .[0]' "${META}") = "${JUST}" ] [ $(jq -r '.context."MY_CONTEXT"' "${META}") = "TeStCoNtExT" ] TO_BUILD="${INVOCATION_DIR}/to-build.json" cat "${TO_BUILD}" echo [ $(jq -r '."upper.txt".type' "${TO_BUILD}") = "ACTION" ] ACTION_ID="$(jq -r '."upper.txt".data.id' "${TO_BUILD}")" GRAPH="${INVOCATION_DIR}/graph.json" cat "${GRAPH}" echo [ $(jq -r '.actions."'"${ACTION_ID}"'".input."data.txt".type' "${GRAPH}") = "LOCAL" ] [ $(jq -r '.actions."'"${ACTION_ID}"'".output | .[0]' "${GRAPH}") = "upper.txt" ] [ $(jq -r '.actions."'"${ACTION_ID}"'".origins | .[0] | .target | .[3]' "${GRAPH}") = "upper" ] PLAIN="${INVOCATION_DIR}/plain.json" cat "${PLAIN}" echo [ $(jq -r '.actions."'"${ACTION_ID}"'".input."data.txt".type' "${PLAIN}") = "LOCAL" ] [ $(jq -r '.actions."'"${ACTION_ID}"'".output | .[0]' "${PLAIN}") = "upper.txt" ] [ $(jq -r '.actions."'"${ACTION_ID}"'".origins' "${PLAIN}") = "null" ] echo # Build again, this time the action should be cached; graph and artifact to # build should not have changed # again abuse the project id to distingush the runs cat > rc.json <<'EOF' {"invocation log": {"project id": "second-run"}} EOF "${JUST_MR}" --rc "${RC}" build upper 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/second-run/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" [ $(jq '.actions | .[] | .cached' "${PROFILE}") = "true" ] TO_BUILD="${INVOCATION_DIR}/to-build.json" cat "${TO_BUILD}" echo [ $(jq -r '."upper.txt".data.id' "${TO_BUILD}") = "${ACTION_ID}" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/failing.sh000066400000000000000000000050371516554100600261420ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${PWD}/log" readonly ETC_DIR="${PWD}/etc" readonly WRK_DIR="${PWD}/work" readonly OUT="${TEST_TMPDIR}/out" # Set up an rc file, requesting invocation logging mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" < repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "will-fail": { "type": "generic" , "outs": ["out.txt"] , "cmds": [ "echo StdOuT" , "echo StdErR 1>&2" , "exit 42" ] } } EOF cat > rc.json <<'EOF' {"invocation log": {"project id": "failing"}} EOF "${JUST_MR}" --rc "${RC}" build will-fail 2>&1 && exit 1 || : INVOCATION_DIR="$(ls -d "${LOG_DIR}"/failing/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # exit code of just should be reported as 1 [ $(jq '."exit code"' "${PROFILE}") -eq 1 ] # exit code of the single action should be reported as 42 [ $(jq -r '.actions | .[] | ."exit code"' "${PROFILE}" 2>&1) -eq 42 ] # stdout and stderr of the single action should be reported correctly STDOUT=$(jq -r '.actions | .[] | .stdout' "${PROFILE}") STDERR=$(jq -r '.actions | .[] | .stderr' "${PROFILE}") "${JUST_MR}" --rc "${RC}" install-cas -o "${OUT}/stdout" "${STDOUT}" 2>&1 "${JUST_MR}" --rc "${RC}" install-cas -o "${OUT}/stderr" "${STDERR}" 2>&1 grep StdOuT "${OUT}/stdout" grep StdErR "${OUT}/stderr" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/profile/time.sh000066400000000000000000000070551516554100600254710ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG_DIR="${PWD}/log" readonly ETC_DIR="${PWD}/etc" readonly WRK_DIR="${PWD}/work" # Set up an rc file, requesting invocation logging mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" <> "${RC}" <> "${RC}" cat "${RC}" # Setup basic project, setting project id mkdir -p "${WRK_DIR}" cd "${WRK_DIR}" touch ROOT cat > repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "": { "type": "generic" , "arguments_config": ["SLEEP"] , "outs": ["done"] , "cmds": [ { "type": "join_cmd" , "$1": [ "sleep" , { "type": "json_encode" , "$1": {"type": "var", "name": "SLEEP", "default": 1} } ] } , "touch done" ] } } EOF # Build for 3 seconds; abuse the project id to distinguish runs cat > rc.json <<'EOF' {"invocation log": {"project id": "3s"}} EOF "${JUST_MR}" --rc "${RC}" -D '{"SLEEP": 3}' build 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/3s/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # we expect non cached, and an action duration at least 3s [ $(jq '.actions | .[] | .cached' "${PROFILE}") = "false" ] [ "$(jq '.actions | .[] | .duration | . >= 3' "${PROFILE}")" = "true" ] # Build for 3 seconds, again; abuse the project id to distinguish runs cat > rc.json <<'EOF' {"invocation log": {"project id": "3s-again"}} EOF "${JUST_MR}" --rc "${RC}" -D '{"SLEEP": 3}' build 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/3s-again/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # we expect cached and no duration specified [ $(jq '.actions | .[] | .cached' "${PROFILE}") = "true" ] [ $(jq '.actions | .[] | .duration' "${PROFILE}") = "null" ] # Build for 4 seconds; abuse the project id to distinguish runs cat > rc.json <<'EOF' {"invocation log": {"project id": "4s"}} EOF "${JUST_MR}" --rc "${RC}" -D '{"SLEEP": 4}' build 2>&1 INVOCATION_DIR="$(ls -d "${LOG_DIR}"/4s/*)" PROFILE="${INVOCATION_DIR}/profile.json" cat "${PROFILE}" # we expect non cached, and an action duration at least 4s [ $(jq '.actions | .[] | .cached' "${PROFILE}") = "false" ] [ "$(jq '.actions | .[] | .duration | . >= 4' "${PROFILE}")" = "true" ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/000077500000000000000000000000001516554100600260245ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/TARGETS000066400000000000000000000065351516554100600270710ustar00rootroot00000000000000{ "native-protocol": { "type": ["end-to-end", "with remote"] , "name": ["native-protocol"] , "test": ["native-protocol.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "large-blobs": { "type": ["end-to-end", "with remote"] , "name": ["large-blobs"] , "test": ["large-blobs.sh"] , "deps": [["", "tool-under-test"]] } , "upload-test": { "type": ["end-to-end", "with remote"] , "name": ["upload-test"] , "test": ["upload-test.sh"] , "deps": [["", "tool-under-test"]] } , "install": { "type": ["end-to-end", "with remote"] , "name": ["install"] , "test": ["install.sh"] , "deps": [["", "tool-under-test"]] } , "execute": { "type": ["end-to-end", "with remote"] , "name": ["execute"] , "test": ["execute.sh"] , "deps": [["", "tool-under-test"]] } , "install-cas": { "type": ["end-to-end", "with remote"] , "name": ["install-cas"] , "test": ["install-cas.sh"] , "deps": [["", "tool-under-test"]] , "keep": [ "out/stdout/remote" , "out/stdout/remote-raw" , "out/stdout/local" , "out/stdout/local-raw" ] } , "install-cas-local": { "type": ["end-to-end", "with remote"] , "name": ["install-cas-local"] , "test": ["install-cas-local.sh"] , "deps": [["", "tool-under-test"]] } , "dispatch": { "type": ["end-to-end", "with remote"] , "name": ["dispatch"] , "test": ["dispatch.sh"] , "deps": [["", "tool-under-test"]] } , "split-splice (raw)": { "type": ["end-to-end", "with remote"] , "name": ["split-splice"] , "test": ["split-splice.sh"] , "deps": [["", "tool-under-test"]] } , "split-splice": { "type": "configure" , "tainted": ["test"] , "target": "split-splice (raw)" , "arguments_config": ["TIMEOUT_SCALE"] , "config": { "type": "singleton_map" , "key": "TIMEOUT_SCALE" , "value": { "type": "*" , "$1": [10, {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0}] } } } , "add-to-cas": { "type": ["end-to-end", "with remote"] , "name": ["add-to-cas"] , "test": ["add-to-cas.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "tree-ops": { "type": ["end-to-end", "with remote"] , "name": ["tree-ops"] , "test": ["tree-ops.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "keep": ["out/graph.json", "out/artifacts.json", "out/log"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["remote-execution"] , "arguments_config": ["TEST_COMPATIBLE_REMOTE", "TEST_BOOTSTRAP_JUST_MR", "DROP_IO_HEAVY_TESTS"] , "deps": { "type": "++" , "$1": [ [ "native-protocol" , "large-blobs" , "upload-test" , "install" , "install-cas" , "install-cas-local" , "dispatch" , "execute" ] , { "type": "if" , "cond": { "type": "or" , "$1": [ {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} ] } , "then": [] , "else": ["add-to-cas"] } , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": ["tree-ops"] } , { "type": "if" , "cond": {"type": "var", "name": "DROP_IO_HEAVY_TESTS"} , "else": ["split-splice"] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/add-to-cas.sh000077500000000000000000000153451516554100600303070ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly ROOT="${PWD}" readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly CLIENT_A="${TEST_TMPDIR}/local-build-root-A" readonly CLIENT_B="${TEST_TMPDIR}/local-build-root-B" readonly CLIENT_SYMLINK_CHECK="${TEST_TMPDIR}/local-build-root-symlink-check" readonly RESOLVE_SPECIAL_CHECK_A="${TEST_TMPDIR}/local-build-root-resolve-special-A" readonly RESOLVE_SPECIAL_CHECK_B="${TEST_TMPDIR}/local-build-root-resolve-special-B" readonly RESOLVE_SPECIAL_CHECK_C="${TEST_TMPDIR}/local-build-root-resolve-special-C" readonly RESOLVE_SPECIAL_CHECK_D="${TEST_TMPDIR}/local-build-root-resolve-special-D" readonly RESOLVE_SPECIAL_CHECK_E="${TEST_TMPDIR}/local-build-root-resolve-special-E" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly OUT_A="${TEST_TMPDIR}/out/client-A" readonly OUT_B="${TEST_TMPDIR}/out/client-B" readonly ARCHIVES_DIR="${TEST_TMPDIR}/archives" readonly LINK_DIR="${TEST_TMPDIR}/add-symlinks-here" REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" # "checkout" of a repository WORK="${ROOT}/work" mkdir work cat > work/TARGETS <<'EOF' { "": { "type": "generic" , "cmds": ["echo -n 'Hello ' > out.txt", "cat name >> out.txt", "echo '!' >> out.txt"] , "outs": ["out.txt"] , "deps": ["name"] } } EOF echo -n World > work/name # There is also an archive with the sources mkdir -p "${ARCHIVES_DIR}" tar cvf "${ARCHIVES_DIR}/src.tar" work echo # Mock vcs tool that should not be called mkdir -p "${TOOLS_DIR}" VCS="${TOOLS_DIR}/mock-vcs" cat > "${VCS}" <<'EOF' #!/bin/sh echo "It should not be necessary to call the foreign VCS" exit 1 EOF chmod 755 "${VCS}" # Preliminaries: verifying link hashing and --follow-symlinks mkdir -p "${LINK_DIR}" ln -s "${WORK}" "${LINK_DIR}/dir" dir_link_hash=$("${JUST}" add-to-cas --local-build-root "${CLIENT_SYMLINK_CHECK}" "${LINK_DIR}/dir") dir_link="$("${JUST}" install-cas --local-build-root "${CLIENT_SYMLINK_CHECK}" "$dir_link_hash")" [ "$dir_link" = "${WORK}" ] dir_follow=$("${JUST}" add-to-cas --local-build-root "${CLIENT_SYMLINK_CHECK}" --follow-symlinks "${LINK_DIR}/dir") dir_hash=$("${JUST}" add-to-cas --local-build-root "${CLIENT_SYMLINK_CHECK}" --follow-symlinks "${WORK}") [ "$dir_follow" = "$dir_hash" ] # Use case: "git tree" repositories and avoiding an additional # call to the foreign version-control system cd ${ROOT} tree=$("${JUST}" add-to-cas --local-build-root "${CLIENT_A}" work) rm -rf work mkdir "${ROOT}/build-A" cd "${ROOT}/build-A" touch ROOT cat > repos.json <&1 echo grep World "${OUT_A}/out.txt" echo # Use case: Backing up a distfile to the remote-execution service cd "${ROOT}" archive=$("${JUST}" add-to-cas --local-build-root "${CLIENT_A}" \ ${REMOTE_EXECUTION_ARGS} "${ARCHIVES_DIR}/src.tar") rm -f "${ARCHIVES_DIR}/src.tar" mkdir -p "${ROOT}/build-B" cd "${ROOT}/build-B" touch ROOT cat > repos.json <&1 echo grep World "${OUT_B}/out.txt" echo # Extension: Using option --resolve-special NUM=5 TREE_ROOT="${ROOT}/tree" mkdir -p "${TREE_ROOT}" ( # create dirs and links structure DIR="${TREE_ROOT}" for i in `seq 1 $NUM`; do TO="." for j in `seq $i $NUM`; do TO="${TO}/$j" # multiple links pointing to same place for k in `seq 1 $NUM`; do x=$(($j + ($k - 1) * $NUM)) ln -s "${TO}" "${DIR}/link$x" done done DIR="${DIR}/$i" mkdir "${DIR}" done # create 1 single file echo content > "${DIR}/data" ) # whether symlinks ignored, resolved inside tree or resolved completely: # expect 6 trees and 2 blobs (1 for data, 1 for exec properties description) "${JUST}" add-to-cas --local-build-root "${RESOLVE_SPECIAL_CHECK_A}" --resolve-special=ignore "${TREE_ROOT}" 2>&1 [ "$(find "${RESOLVE_SPECIAL_CHECK_A}/protocol-dependent/generation-0" -type f | grep cast | wc -l)" = $(($NUM + 1)) ] [ "$(find "${RESOLVE_SPECIAL_CHECK_A}/protocol-dependent/generation-0" -type f | grep casf | wc -l)" = 2 ] "${JUST}" add-to-cas --local-build-root "${RESOLVE_SPECIAL_CHECK_B}" --resolve-special=tree-all "${TREE_ROOT}" 2>&1 [ "$(find "${RESOLVE_SPECIAL_CHECK_B}/protocol-dependent/generation-0" -type f | grep cast | wc -l)" = $(($NUM + 1)) ] [ "$(find "${RESOLVE_SPECIAL_CHECK_B}/protocol-dependent/generation-0" -type f | grep casf | wc -l)" = 2 ] "${JUST}" add-to-cas --local-build-root "${RESOLVE_SPECIAL_CHECK_C}" --resolve-special=all "${TREE_ROOT}" 2>&1 [ "$(find "${RESOLVE_SPECIAL_CHECK_C}/protocol-dependent/generation-0" -type f | grep cast | wc -l)" = $(($NUM + 1)) ] [ "$(find "${RESOLVE_SPECIAL_CHECK_C}/protocol-dependent/generation-0" -type f | grep casf | wc -l)" = 2 ] ( # create upward symlinks DIR="${TREE_ROOT}" for i in `seq 1 $NUM`; do ln -s "../link" "${DIR}/link" DIR="${DIR}/$i" done # link the outmost entry to the data ln -s "${DIR}/data" "${ROOT}/link" ) # upward symlinks cannot be resolved inside tree... "${JUST}" add-to-cas --local-build-root "${RESOLVE_SPECIAL_CHECK_D}" \ --resolve-special=tree-all "${TREE_ROOT}" 2>&1 && echo expected to fail || : echo failed as expected echo # ...but they can get resolved outside of it; expect same number of CAS entries "${JUST}" add-to-cas --local-build-root "${RESOLVE_SPECIAL_CHECK_E}" --resolve-special=all "${TREE_ROOT}" 2>&1 [ "$(find "${RESOLVE_SPECIAL_CHECK_E}/protocol-dependent/generation-0" -type f | grep cast | wc -l)" = $(($NUM + 1)) ] [ "$(find "${RESOLVE_SPECIAL_CHECK_E}/protocol-dependent/generation-0" -type f | grep casf | wc -l)" = 2 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/dispatch.sh000077500000000000000000000066301516554100600301670ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBRDIR="${PWD}/local-build-root" readonly ESDIR="${PWD}/service-build-root" readonly INFOFILE="${PWD}/info.json" readonly PIDFILE="${PWD}/pid.txt" # A standard remote-execution server is given by the test infrastructure REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" LOCAL_ARGS="" if [ -n "${COMPATIBLE:-}" ] then REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} --compatible" LOCAL_ARGS="--compatible" fi # To test the dispatch, we have a remote-execution server # with a special variant of cat that # - prefixes with some extra string, but # - only takes the first argument into account. # In this way, we can distinguish it from the regular cat on the # on the normal remote execution server. readonly REFERENCE_OUTPUT="FooOOOooo" readonly SERVER_BIN_DIR="${TMPDIR}/server/bin" mkdir -p "${SERVER_BIN_DIR}" cat > "${SERVER_BIN_DIR}/cat" <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") cat > dispatch.json < TARGETS <<'EOF' { "payload": { "type": "generic" , "cmds": ["echo this-is-the-payload > payload.txt"] , "outs": ["payload.txt"] } , "drop": { "type": "generic" , "cmds": ["echo please-drop-this > drop.txt"] , "outs": ["drop.txt"] } , "post": { "type": "generic" , "cmds": ["echo this-is-added-at-the-end > post.txt"] , "outs": ["post.txt"] } , "special-dispatch": { "type": "generic" , "cmds": ["cat payload.txt drop.txt > out.txt"] , "outs": ["out.txt"] , "execution properties": {"type": "singleton_map", "key": "server", "value": "special"} , "deps": ["payload", "drop"] } , "": { "type": "generic" , "cmds": ["cat out.txt post.txt > final.txt"] , "deps": ["special-dispatch", "post"] , "outs": ["final.txt"] } } EOF FAILED="" "${JUST}" install -o . --local-build-root "${LBRDIR}" \ ${REMOTE_EXECUTION_ARGS} --endpoint-configuration dispatch.json 2>&1 \ || FAILED=YES kill $(cat "${PIDFILE}") [ -z "${FAILED}" ] [ -f final.txt ] echo echo content of final.txt cat final.txt echo grep ${REFERENCE_OUTPUT} final.txt grep this-is-the-payload final.txt grep drop final.txt && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/execute.sh000066400000000000000000000034611516554100600300260ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" LOCAL_ARGS="" if [ -n "${COMPATIBLE:-}" ] then REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} --compatible" LOCAL_ARGS="--compatible" fi mkdir work cd work touch ROOT for i in `seq 1 32` do cat > generate-$i.sh < \$1 EOF chmod 755 generate-$i.sh done cat > TARGETS <> TARGETS <> TARGETS for i in `seq 1 32` do cat >> TARGETS <> TARGETS cat TARGETS echo # remote build should succeed "${JUST}" build --local-build-root "${LBR}" \ ${REMOTE_EXECUTION_ARGS} --dump-artifacts remote.json 2>&1 echo # local build should succeed, and yield the same result "${JUST}" build --local-build-root "${LBR}" \ ${LOCAL_ARGS} --dump-artifacts local.json 2>&1 echo diff remote.json local.json echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/install-cas-local.sh000066400000000000000000000064301516554100600316650ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBRDIR="${TEST_TMPDIR}/local-build-root" readonly SETUP_LBRDIR="${TEST_TMPDIR}/build-root-for-setup-only" readonly FILES_DIR="${TEST_TMPDIR}/data" readonly OUT="${TEST_TMPDIR}/out" mkdir -p "${FILES_DIR}" mkdir -p "${OUT}" LOCAL_ARGS="" REMOTE_ARGS="${LOCAL_ARGS} -r ${REMOTE_EXECUTION_ADDRESS}" if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" LOCAL_ARGS="${LOCAL_ARGS} --compatible" fi # Set up secanrio, by creating a hash that is only known remotely # and one that is only known locally. Moreover, compute a hash of # a file unknown to both. readonly REMOTE_FILE="${FILES_DIR}/remote" echo 'This file is only known to ReMoTe!!' > "${REMOTE_FILE}" REMOTE_HASH=$("${JUST}" add-to-cas --local-build-root "${SETUP_LBRDIR}" ${REMOTE_ARGS} "${REMOTE_FILE}") echo "Remote file has hash ${REMOTE_HASH}" readonly LOCAL_FILE="${FILES_DIR}/local" echo 'This file is only known LoCaLlY ...' > "${LOCAL_FILE}" LOCAL_HASH=$("${JUST}" add-to-cas --local-build-root "${LBRDIR}" ${LOCAL_ARGS} "${LOCAL_FILE}") echo "Local file has hash ${LOCAL_HASH}" readonly UNKNOWN_FILE="${FILES_DIR}/unknown" echo '... for Rumpelstiltskin is my name' > "${UNKNOWN_FILE}" UNKNOWN_HASH=$("${JUST}" add-to-cas --local-build-root "${SETUP_LBRDIR}" ${LOCAL_ARGS} "${UNKNOWN_FILE}") echo "Unkown file has hash ${UNKNOWN_HASH}" # Verify that with the canonical arguments (as, e.g., just-mr might provide them # with appropriate rc file) we can get both files correctly. Also verify that # we get an exit code identifying an error if we failed to fetch the file. echo remote, via stdout "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ "${REMOTE_HASH}" > "${OUT}/remote" cmp "${OUT}/remote" "${REMOTE_FILE}" rm -f "${OUT}/remote" echo remote, specified output file "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ -o "${OUT}/remote" "${REMOTE_HASH}" 2>&1 cmp "${OUT}/remote" "${REMOTE_FILE}" echo local, via stdout "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ "${LOCAL_HASH}" > "${OUT}/local" cmp "${OUT}/local" "${LOCAL_FILE}" rm -f "${OUT}/local" echo local, specified output file "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ -o "${OUT}/local" "${LOCAL_HASH}" 2>&1 cmp "${OUT}/local" "${LOCAL_FILE}" echo unknown, via stdout "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ "${UNKNOWN_HASH}" 2>&1 && exit 1 || : echo unknown, specified output path "${JUST}" install-cas --local-build-root "${LBRDIR}" ${REMOTE_ARGS} \ -o "${OUT}/unknown" "${UNKNOWN_HASH}" 2>&1 && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/install-cas.sh000066400000000000000000000050371516554100600305770ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly LBRDIR="${TEST_TMPDIR}/local-build-root" readonly OUTDIR="${PWD}/out" LOCAL_ARGS="--local-build-root ${LBRDIR}" REMOTE_ARGS="${LOCAL_ARGS} -r ${REMOTE_EXECUTION_ADDRESS}" REMOTE_PROPS="" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ] then REMOTE_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" fi if [ -n "${COMPATIBLE:-}" ] then REMOTE_ARGS="${REMOTE_ARGS} --compatible" LOCAL_ARGS="${LOCAL_ARGS} --compatible" fi # Build a tree remotely and get its identifier mkdir src cd src touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "out_dirs": ["out"] , "cmds": [ "mkdir -p out/deep/inside/path" , "echo Hello World > out/deep/inside/path/hello.txt" ] } } EOF "${JUST}" build ${REMOTE_ARGS} ${REMOTE_PROPS} --dump-artifacts artifacts.json 2>&1 echo cat artifacts.json OUT="$(jq -r '.out.id' artifacts.json)::t" echo $OUT echo # install to stdout in all possible ways mkdir -p "${OUTDIR}/stdout" "${JUST}" install-cas ${REMOTE_ARGS} --raw-tree "${OUT}" > "${OUTDIR}/stdout/remote-raw" "${JUST}" install-cas ${REMOTE_ARGS} --remember "${OUT}" > "${OUTDIR}/stdout/remote" "${JUST}" install-cas ${LOCAL_ARGS} "${OUT}" --raw-tree > "${OUTDIR}/stdout/local-raw" "${JUST}" install-cas ${LOCAL_ARGS} "${OUT}" > "${OUTDIR}/stdout/local" # verify consistency between local and remote values cmp "${OUTDIR}/stdout/local-raw" "${OUTDIR}/stdout/remote-raw" cmp "${OUTDIR}/stdout/local" "${OUTDIR}/stdout/remote" # verify the -P option "${JUST}" install-cas ${REMOTE_ARGS} -P deep/inside/path/hello.txt \ -o "${OUTDIR}/path/remote" "${OUT}" 2>&1 "${JUST}" install-cas ${LOCAL_ARGS} -P deep/inside/path/hello.txt \ -o "${OUTDIR}/path/local" "${OUT}" 2>&1 echo ls -alR "${OUTDIR}/path" [ -f "${OUTDIR}/path/remote" ] grep World "${OUTDIR}/path/remote" [ -f "${OUTDIR}/path/local" ] grep World "${OUTDIR}/path/local" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/install.sh000066400000000000000000000135371516554100600300370ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly WRKDIR="${PWD}" readonly LBRDIR_A="${TEST_TMPDIR}/local-build-root/instance-A" readonly LBRDIR_B="${TEST_TMPDIR}/local-build-root/instance-B" readonly LBRDIR_C="${TEST_TMPDIR}/local-build-root/instance-C" readonly SRCDIR_A="${TEST_TMPDIR}/src/instance-A" readonly SRCDIR_B="${TEST_TMPDIR}/src/instance-B" readonly SRCDIR_C="${TEST_TMPDIR}/src/instance-C" readonly OUTDIR_B="${TEST_TMPDIR}/out/instance-B" readonly OUTDIR_C="${TEST_TMPDIR}/out/instance-C" REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" REMOTE_EXECUTION_PROPS="" LOCAL_ARGS="" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ] then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" fi if [ -n "${COMPATIBLE:-}" ] then REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} --compatible" LOCAL_ARGS="--compatible" fi # install-cas from remote execution endpoint ## instance A to upload mkdir -p "${SRCDIR_A}" cd "${SRCDIR_A}" touch ROOT mkdir -p tree/foo/bar mkdir -p tree/baz echo content frist file > tree/foo/bar/data.txt echo content second file > tree/baz/data.txt ln -s dummy tree/foo/link cat > TARGETS <<'EOF' {"": {"type": "install", "dirs": [[["TREE", null, "tree"], "."]]}} EOF "${JUST}" build --local-build-root "${LBRDIR_A}" \ --dump-artifacts "${WRKDIR}/tree.json" \ ${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS} 2>&1 ## instance B should be able to fetch cd "${WRKDIR}" TREE_ID=$(jq -r '.tree.id' "${WRKDIR}/tree.json")"::t" "${JUST}" install-cas --local-build-root "${LBRDIR_B}" \ -o "${OUTDIR_B}/first" ${REMOTE_EXECUTION_ARGS} "${TREE_ID}" 2>&1 for f in $(cd "${SRCDIR_A}/tree" && find . -type f) do cmp "${SRCDIR_A}/tree/$f" "${OUTDIR_B}/first/$f" done for l in $(cd "${SRCDIR_A}/tree" && find . -type L) do cmp $(readlink "${SRCDIR_A}/tree/$l") $(readlink "${OUTDIR_B}/first/$l") done ## ... and to remember "${JUST}" install-cas --local-build-root "${LBRDIR_B}" \ -o "${OUTDIR_B}/first-discard" \ --remember ${REMOTE_EXECUTION_ARGS} "${TREE_ID}" 2>&1 "${JUST}" install-cas --local-build-root "${LBRDIR_B}" \ -o "${OUTDIR_B}/first-from-local" ${LOCAL_ARGS} "${TREE_ID}" 2>&1 for f in $(cd "${SRCDIR_A}/tree" && find . -type f) do cmp "${SRCDIR_A}/tree/$f" "${OUTDIR_B}/first-from-local/$f" done for l in $(cd "${SRCDIR_A}/tree" && find . -type L) do cmp $(readlink "${SRCDIR_A}/tree/$l") $(readlink "${OUTDIR_B}/first-from-local/$l") done # install-cas has to prefer (at least: use) local CAS, also with remote endpoint ## instance B builds locally, to fill local CAS ## (using a different tree, not known to the remote execution) mkdir -p "${SRCDIR_B}" cd "${SRCDIR_B}" touch ROOT mkdir -p tree/different/foobar mkdir -p tree/bar/other_dir mkdir -p tree/another_dir echo some other content > tree/different/foobar/file.txt echo yet another content > tree/bar/other_dir/file.txt ln -s dummy tree/another_dir/link cat > TARGETS <<'EOF' {"": {"type": "install", "dirs": [[["TREE", null, "tree"], "."]]}} EOF "${JUST}" build --local-build-root "${LBRDIR_B}" \ --dump-artifacts "${WRKDIR}/tree.json" ${LOCAL_ARGS} 2>&1 ## install-cas should work, even though the tree is not known remotely cd "${WRKDIR}" TREE_ID=$(jq -r '.tree.id' "${WRKDIR}/tree.json")"::t" "${JUST}" install-cas --local-build-root "${LBRDIR_B}" \ -o "${OUTDIR_B}/second" ${REMOTE_EXECUTION_ARGS} "${TREE_ID}" 2>&1 for f in $(cd "${SRCDIR_B}/tree" && find . -type f) do cmp "${SRCDIR_B}/tree/$f" "${OUTDIR_B}/second/$f" done for l in $(cd "${SRCDIR_B}/tree" && find . -type L) do cmp $(readlink "${SRCDIR_B}/tree/$l") $(readlink "${OUTDIR_B}/second/$l") done # install --remember ## install a tree with --remember, this also tests blob splitting for: ## - file smaller than minimum chunk size (2 KB) ## - file larger than maximum chunk size (64 KB) ## - tree smaller than minimum chunk size ## - tree larger than maximum chunk size mkdir -p "${SRCDIR_C}" cd "${SRCDIR_C}" touch ROOT cat > TARGETS <<'EOF' { "": { "type": "generic" , "out_dirs": ["out"] , "cmds": [ "mkdir -p out/foo out/bar out/baz out/large" , "echo some file content > out/foo/data.txt" , "echo more file content > out/bar/file.txt" , "echo even more file content > out/bar/another_file.txt" , "ln -s dummy out/baz/link" , "for i in $(seq 1000); do echo foo > out/large/$(printf %064d $i).txt; done" , "for i in $(seq 128); do seq 1 1024 >> out/large.txt; done" ] } } EOF "${JUST}" install --local-build-root "${LBRDIR_C}" \ --remember -o "${OUTDIR_C}/remote" \ --dump-artifacts "${WRKDIR}/tree.json" \ ${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS} 2>&1 ## now it should also be available for local installation TREE_ID=$(jq -r ".\"${OUTDIR_C}/remote/out\".id" "${WRKDIR}/tree.json")"::t" "${JUST}" install-cas --local-build-root "${LBRDIR_C}" \ -o "${OUTDIR_C}/local" \ ${LOCAL_ARGS} "${TREE_ID}" 2>&1 for f in $(cd "${OUTDIR_C}/remote/out" && find . -type f) do cmp "${OUTDIR_C}/remote/out/$f" "${OUTDIR_C}/local/$f" done for l in $(cd "${OUTDIR_C}/remote/out" && find . -type L) do cmp $(readlink "${OUTDIR_C}/remote/out/$l") $(readlink "${OUTDIR_C}/local/$l") done echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/large-blobs.sh000066400000000000000000000047701516554100600305610ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" # create a sufficiently large (>4MB) file for testing upload/download (16MB) dd if=/dev/zero of=large.file bs=1024 count=$((16*1024)) touch ROOT cat > TARGETS < "${DIRNAME}/foo.txt" echo bar > "${DIRNAME}/bar.txt" git add "${DIRNAME}" git commit -m "Initial commit" readonly GIT_COMMIT_ID="$(git log -n 1 --format=%H)" readonly GIT_TREE_ID="$(git log -n 1 --format=%T)" cat > repos.json < TARGETS <&1 TREE_ID="$(jq -r ".${OUT_DIRNAME}.id" "${RESULT}" 2>&1)" test ${TREE_ID} ${EQUAL} ${GIT_TREE_ID} REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS}" fi echo echo Upload and download Git tree to remote CAS in ${NAME} mode echo "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' -C "${CONF}" --main test \ ${REMOTE_EXECUTION_ARGS} --local-build-root="${LBRDIR}" \ --dump-artifacts "${RESULT}" ${ARGS} test 2>&1 TREE_ID="$(jq -r ".${OUT_DIRNAME}.id" "${RESULT}" 2>&1)" test ${TREE_ID} ${EQUAL} ${GIT_TREE_ID} just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/split-splice.sh000066400000000000000000000100411516554100600307640ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" # Local readonly LOCAL_ROOT="${TMPDIR}/local" readonly LOCAL_CACHE="${LOCAL_ROOT}/build-root" # Remote 1 readonly R1_ROOT=$(readlink -f "${PWD}/../remote") readonly R1_CACHE="${R1_ROOT}/build-root" readonly R1_ADDRESS="${REMOTE_EXECUTION_ADDRESS}" # Remote 2 readonly R2_ROOT="${TMPDIR}/remote" readonly R2_CACHE="${R2_ROOT}/build-root" readonly R2_INFOFILE="${R2_ROOT}/info.json" readonly R2_PIDFILE="${R2_ROOT}/pid.txt" readonly R2_BIN_DIR="${R2_ROOT}/bin" COMPATIBLE_ARGS="" if [ -n "${COMPATIBLE:-}" ]; then COMPATIBLE_ARGS="--compatible" fi # This test tests the round trip of large objects being split at one # remote execution, being spliced at another remote execution, and being # transferred back to the host. # # This test requires two remote-execution servers. One is given # implicitly by the test infrastructure, another one is explicitly # started. # # In order to verify the round trip, we add a magic word into the # objects created at remote execution 1, make a transformation to upper # case at remote execution 2 using a special local launcher, and grep # for the modified magic word at the local host. mkdir -p "${R2_BIN_DIR}" cat > "${R2_BIN_DIR}/toupper" <<'EOF' #!/bin/sh tr 'a-z' 'A-Z' < "$1" EOF chmod 755 "${R2_BIN_DIR}/toupper" echo R2_LAUNCHER='["env", "PATH='"${R2_BIN_DIR}:${PATH}"'"]' echo "Remote-execution 2 will use ${R2_LAUNCHER} as local launcher" echo "${JUST}" execute --info-file "${R2_INFOFILE}" --pid-file "${R2_PIDFILE}" \ --local-build-root "${R2_CACHE}" -L "${R2_LAUNCHER}" \ ${COMPATIBLE_ARGS} 2>&1 & for _ in `seq 1 60`; do if test -f "${R2_INFOFILE}"; then break fi sleep 1; done if ! test -f "${R2_INFOFILE}"; then echo "Did not find ${R2_INFOFILE}" exit 1 fi cleanup() { kill $(cat "${R2_PIDFILE}") } trap cleanup EXIT readonly R2_PORT=$(jq '."port"' "${R2_INFOFILE}") readonly R2_ADDRESS="127.0.0.1:${R2_PORT}" echo echo "Remote-execution 1: ${R1_ADDRESS}" echo "Remote-execution 2: ${R2_ADDRESS}" echo # A file and a tree > 4 MB each are created at remote execution 1, sent # to the remote execution 2 via a dispatch action, enforcing splitting # at remote execution 1 and splicing at remote-execution 2. The final # result is sent back to the host. cat > TARGETS <<'EOF' { "file": { "type": "generic" , "outs": ["file.txt"] , "cmds": [ "for i in $(seq 21846); do printf '%0192d\n' $i >> file.txt; done" , "echo magic_word >> file.txt" ] } , "tree": { "type": "generic" , "out_dirs": ["tree"] , "cmds": [ "mkdir -p tree" , "for i in $(seq 21846); do echo foo > tree/$(printf '%0192d' $i).txt; done" , "echo foo > tree/magic_word.txt" ] } , "dispatch": { "type": "generic" , "cmds": [ "toupper file.txt > file_out.txt" , "ls tree > tree.txt" , "toupper tree.txt > tree_out.txt" ] , "outs": ["file_out.txt", "tree_out.txt"] , "execution properties": {"type": "singleton_map", "key": "server", "value": "special"} , "deps": ["file", "tree"] } , "": { "type": "generic" , "cmds": ["cat file_out.txt tree_out.txt > final.txt"] , "deps": ["dispatch"] , "outs": ["final.txt"] } } EOF cat > dispatch.json <&1 [ "$(grep -c MAGIC_WORD final.txt)" = 2 ] echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/tree-ops.sh000066400000000000000000000063311516554100600301210ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBR="${TMPDIR}/local-build-root" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly ETC_DIR="${ROOT}/etc" readonly WRKDIR="${ROOT}/work" mkdir -p "${ETC_DIR}" readonly RC="${ETC_DIR}/rc.json" cat > "${RC}" <> "${RC}" <> "${RC}" < repos.json <<'EOF' {"repositories": {"": {"repository": {"type": "file", "path": "."}}}} EOF cat > TARGETS <<'EOF' { "early": { "type": "generic" , "out_dirs": ["out"] , "cmds": [ "mkdir -p out/foo out/mixed" , "for i in `seq 1 20`; do echo FOO > out/foo/data$i.txt; done" , "for i in `seq 1 19`; do echo early > out/mixed/data$i.txt; done" ] } , "late": { "type": "generic" , "out_dirs": ["out"] , "cmds": [ "mkdir -p out/bar out/mixed" , "for i in `seq 1 20`; do echo BAR > out/bar/data$i.txt; done" , "for i in `seq 10 30`; do echo late > out/mixed/data$i.txt; done" ] } , "overlay": {"type": "tree_overlay", "name": "overlay", "deps": ["early", "late"]} , "TheOffendingTarget": { "type": "disjoint_tree_overlay" , "name": "overlay" , "deps": ["early", "late"] } , "conflict": {"type": "install", "dirs": [["TheOffendingTarget", ""]]} } EOF # Conflict-free build "${JUST_MR}" --rc "${RC}" install -o "${OUT}/overlay" overlay 2>&1 grep FOO "${OUT}/overlay/overlay/out/foo/data15.txt" grep BAR "${OUT}/overlay/overlay/out/bar/data15.txt" grep early "${OUT}/overlay/overlay/out/mixed/data5.txt" grep late "${OUT}/overlay/overlay/out/mixed/data15.txt" grep late "${OUT}/overlay/overlay/out/mixed/data25.txt" # Analysis of conflict should work "${JUST_MR}" --rc "${RC}" analyse \ --dump-graph "${OUT}/graph.json" \ --dump-artifacts-to-build "${OUT}/artifacts.json" \ conflict 2>&1 echo [ "$(jq -r '.overlay.type' "${OUT}/artifacts.json")" = TREE_OVERLAY ] OVERLAY_ID=$(jq -r '.overlay.data.id' "${OUT}/artifacts.json") [ "$(jq '."tree_overlays"."'"${OVERLAY_ID}"'".trees | length' "${OUT}/graph.json")" -eq 2 ] # Building the tree conflict should fail with a reasonable error message "${JUST_MR}" --rc "${RC}" build \ -f "${OUT}/log" conflict 2>&1 && exit 1 || : grep 'data1..txt' "${OUT}/log" grep 'TheOffendingTarget' "${OUT}/log" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/remote-execution/upload-test.sh000066400000000000000000000056421516554100600306300ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly GITDIR="${TEST_TMPDIR}/src" readonly LBRDIR_1="${TEST_TMPDIR}/local-build-root-1" readonly LBRDIR_2="${TEST_TMPDIR}/local-build-root-2" mkdir -p ${GITDIR} cd ${GITDIR} git init git config user.name "Nobody" git config user.email "nobody@example.org" mkdir -p foo/bar/baz echo `hostname`.`date +%s`.$$ > foo/bar/baz/data.txt ln -s dummy foo/bar/baz/link cat > TARGETS <<'EOF' { "": { "type": "generic" , "outs": ["out.txt", "newlink"] , "cmds": ["find . > out.txt", "ln -s dummy newlink"] , "deps": [["TREE", null, "."]] } } EOF git add . git commit -m 'Generated new data' readonly TREE=$(git log --pretty=%T) cd ${TEST_TMPDIR} echo === regular root === cat > repos.json <&1 # Build remotely REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS}" fi "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' -C "${CONF}" --local-build-root="${LBRDIR_1}" ${ARGS} ${REMOTE_EXECUTION_ARGS} 2>&1 echo === ignore_special root === cat > repos.json <&1 # Build remotely REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS}" fi "${JUST}" build -L '["env", "PATH='"${PATH}"'"]' -C "${CONF}" --local-build-root="${LBRDIR_2}" ${ARGS} ${REMOTE_EXECUTION_ARGS} 2>&1 just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/000077500000000000000000000000001516554100600253125ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/TARGETS000066400000000000000000000203231516554100600263460ustar00rootroot00000000000000{ "serve-target-remote-build": { "type": ["end-to-end", "with serve"] , "name": ["serve-target-remote-build"] , "test": ["serve_target_remote_build.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": [ "serve-target-remote-build (repo)" , "serve-target-remote-build (rule)" , "serve-target-remote-build (target)" ] } , "serve-target-failed-build": { "type": ["end-to-end", "with serve"] , "name": ["serve-target-failed-build"] , "test": ["serve_target_failed_build.sh"] , "deps": [ "data/rules/RULES" , "data/rules/RULES.dummy" , "data/targets/TARGETS" , "serve-tree (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": [ "serve-target-remote-build (repo)" , "serve-target-failed-build (rule)" , "serve-target-remote-build (target)" ] } , "serve-target-remote-build (repo)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": [ "mkdir -p src" , "for i in `seq 1 5` ; do echo $i > src/$i.txt ; done" , "mkdir src/subdir" , "for i in `seq 6 10` ; do echo $i > src/subdir/sub-$i.txt ; done" ] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "serve-target-remote-build (target)": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data/targets/TARGETS"] , "stage": ["test", "end-to-end", "serve-service"] } , "serve-target-remote-build (rule)": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data/rules/RULES"] , "stage": ["test", "end-to-end", "serve-service"] } , "serve-target-failed-build (rule)": { "type": ["@", "rules", "data", "staged"] , "srcs": ["data/rules/RULES.dummy"] , "stage": ["test", "end-to-end", "serve-service"] } , "serve-target-cache-hit": { "type": ["end-to-end", "with serve"] , "name": ["serve-target-cache-hit"] , "test": ["serve_target_cache_hit.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "serve-start-execute": { "type": ["@", "rules", "shell/test", "script"] , "name": ["serve-start-execute"] , "test": ["serve_start_execute.sh"] , "deps": [["", "tool-under-test"]] } , "serve-start-execute-sharding": { "type": ["@", "rules", "shell/test", "script"] , "name": ["serve-start-execute-sharding"] , "test": ["serve_start_execute_sharding.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "serve-export-deps": { "type": ["end-to-end", "with serve"] , "name": ["serve-export-deps"] , "test": ["serve_export_deps.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "serve-tree (data)": {"type": "install", "dirs": [["serve-target-remote-build (repo)", "repo"]]} , "serve-tree (archive)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "outs": ["src.tar"] , "cmds": ["tar cf src.tar repo"] , "deps": ["serve-tree (data)"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "serve-tree": { "type": ["end-to-end", "with serve"] , "name": ["serve-tree"] , "test": ["serve-tree.sh"] , "deps": [ "data/targets/TARGETS.tree" , "serve-tree (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-tree (archive)"] } , "serve-target-remote-build-dispatch (data)": { "type": "install" , "tainted": ["test"] , "files": {"TARGETS": "data/targets/TARGETS.dispatch"} } , "serve-target-remote-build-dispatch": { "type": ["end-to-end", "with serve"] , "name": ["serve-target-remote-build-dispatch"] , "test": ["serve_target_remote_build_dispatch.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["serve-target-remote-build-dispatch (data)"] } , "serve-query-target-cache-value": { "type": ["end-to-end", "with serve"] , "name": ["serve-query-target-cache-value"] , "test": ["serve_query_target_cache_value.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "describe (data)": { "type": "install" , "files": {"ROOT": "data/ROOT", "TARGETS": "data/targets/TARGETS.describe"} } , "describe (locally installed)": {"type": "install", "dirs": [["describe (data)", "repo"]]} , "describe": { "type": ["end-to-end", "with serve"] , "name": ["describe"] , "test": ["describe.sh"] , "deps": [ "describe (locally installed)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["describe (data)"] , "keep": ["out/describe.orig", "out/describe"] } , "failure-report (data)": { "type": "install" , "files": {"ROOT": "data/ROOT", "TARGETS": "data/targets/TARGETS.fail"} } , "failure-report": { "type": ["end-to-end", "with serve"] , "name": ["failure-report"] , "test": ["failure-report.sh"] , "deps": [ "describe (locally installed)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["failure-report (data)"] , "keep": ["out/log", "out/serve.log", "out/failure.log"] } , "deep tree (data)": { "type": "install" , "files": { "ROOT": "data/ROOT" , "TARGETS": "data/targets/TARGETS.deep" , "deep-output.py": "data/deep-output.py" } } , "deep tree": { "type": ["end-to-end", "with serve"] , "name": ["deep-tree"] , "test": ["deep-tree.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["deep tree (data)"] , "keep-dirs": ["out"] } , "interruption-test (data)": { "type": "install" , "files": {"ROOT": "data/ROOT", "TARGETS": "data/targets/TARGETS.slow"} } , "interruption-test": { "type": ["end-to-end", "with serve"] , "name": ["interruption-test"] , "test": ["interruption-test.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["interruption-test (data)"] , "keep-dirs": ["out"] } , "load test (data)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "outs": ["ROOT", "TARGETS"] , "cmds": ["touch ROOT", "python3 wide-deep-targets.py"] , "deps": ["wide-deep-targets.py"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "load test": { "type": ["end-to-end", "with serve"] , "name": ["load-test"] , "test": ["load-test.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["load test (data)"] , "keep-dirs": ["out"] } , "serve-many-targets (data)": { "type": "install" , "files": {"ROOT": "data/ROOT", "TARGETS": "data/targets/TARGETS.transform"} } , "serve-many-targets (data in module)": { "type": "install" , "files": {"ROOT": "data/ROOT", "module/TARGETS": "data/targets/TARGETS.transform"} } , "serve-many-targets": { "type": ["end-to-end", "with serve"] , "name": ["serve-many-targets"] , "test": ["serve_many_targets.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["serve-many-targets (data)", "serve-many-targets (data in module)"] } , "deduplication-of-serve-requests": { "type": ["end-to-end", "with serve"] , "name": ["deduplication-of-serve-requests"] , "test": ["deduplication_of_serve_requests.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["serve-many-targets (data)"] } , "TESTS (unconfigured)": { "type": ["@", "rules", "test", "matrix"] , "stage": ["serve-service"] , "deps": [ "deduplication-of-serve-requests" , "deep tree" , "describe" , "failure-report" , "interruption-test" , "load test" , "serve-export-deps" , "serve-many-targets" , "serve-query-target-cache-value" , "serve-start-execute" , "serve-start-execute-sharding" , "serve-target-cache-hit" , "serve-target-failed-build" , "serve-target-remote-build" , "serve-target-remote-build-dispatch" , "serve-tree" , ["./", "serve-archive-root", "TESTS"] , ["./", "serve-distdir-root", "TESTS"] , ["./", "serve-file-root", "TESTS"] , ["./", "serve-git-root", "TESTS"] , ["./", "serve-git-tree-root", "TESTS"] ] } , "TESTS": { "type": "configure" , "tainted": ["test"] , "target": "TESTS (unconfigured)" , "config": { "type": "'" , "$1": { "TEST_MATRIX": { "TEST_STANDALONE_SERVE": {"standalone-serve": true, "separate-remote": false} } } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/000077500000000000000000000000001516554100600262235ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/ROOT000066400000000000000000000000001516554100600267170ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/deep-output.py000066400000000000000000000015741516554100600310570ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys outname = sys.argv[1] depth = int(sys.argv[2]) outpath = os.path.join(outname, *(["x"] * depth)) os.makedirs(outpath) for i in range(10): with open(os.path.join(outpath, "data%d.txt" % (i,)), "w") as f: f.write("%d" % (i,)) just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/rules/000077500000000000000000000000001516554100600273555ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/rules/RULES000066400000000000000000000017571516554100600302040ustar00rootroot00000000000000{ "ls -R": { "target_fields": ["tree"] , "expression": { "type": "let*" , "bindings": [ [ "tree" , { "type": "TREE" , "$1": { "type": "map_union" , "$1": { "type": "foreach" , "var": "dep" , "range": {"type": "FIELD", "name": "tree"} , "body": {"type": "DEP_RUNFILES", "dep": {"type": "var", "name": "dep"}} } } } ] , [ "inputs" , { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "tree" , "value": {"type": "var", "name": "tree"} } ] } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "ACTION" , "outs": ["_out"] , "inputs": {"type": "var", "name": "inputs"} , "cmd": ["sh", "-c", "find . -name '*.txt' > _out"] } } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/rules/RULES.dummy000066400000000000000000000000031516554100600313150ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/000077500000000000000000000000001516554100600276745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS000066400000000000000000000002561516554100600307330ustar00rootroot00000000000000{ "": { "type": "export" , "target": "list-tree" , "flexible_config": ["AR", "ARCH", "ENV", "OS"] } , "list-tree": {"type": "ls -R", "tree": [["TREE", null, "."]]} } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.deep000066400000000000000000000005321516554100600316440ustar00rootroot00000000000000{ "": {"type": "export", "target": "deep", "flexible_config": ["DEPTH"]} , "deep": { "type": "generic" , "arguments_config": ["DEPTH"] , "deps": ["deep-output.py"] , "out_dirs": ["out"] , "cmds": [ { "type": "join_cmd" , "$1": ["python3", "deep-output.py", "out", {"type": "var", "name": "DEPTH"}] } ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.describe000066400000000000000000000010671516554100600325130ustar00rootroot00000000000000{ "": { "type": "export" , "target": "_" , "doc": ["A glorified hello-world example"] , "flexible_config": ["NAME"] , "config_doc": {"NAME": ["The name to say hello to"]} } , "_": { "type": "generic" , "arguments_config": ["NAME"] , "outs": ["greeting.txt"] , "cmds": [ { "type": "join" , "$1": [ { "type": "join_cmd" , "$1": [ "echo" , "Hello" , {"type": "var", "name": "NAME", "default": "world"} ] } , " > greeting.txt" ] } ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.dispatch000066400000000000000000000015411516554100600325270ustar00rootroot00000000000000{ "payload": { "type": "generic" , "cmds": ["echo this-is-the-payload > payload.txt"] , "outs": ["payload.txt"] } , "drop": { "type": "generic" , "cmds": ["echo please-drop-this > drop.txt"] , "outs": ["drop.txt"] } , "post": { "type": "generic" , "cmds": ["echo this-is-added-at-the-end > post.txt"] , "outs": ["post.txt"] } , "special-dispatch": { "type": "generic" , "cmds": ["cat payload.txt drop.txt > out.txt"] , "outs": ["out.txt"] , "execution properties": {"type": "singleton_map", "key": "server", "value": "special"} , "deps": ["drop", "payload"] } , "internal": { "type": "generic" , "cmds": ["cat out.txt post.txt > final.txt"] , "deps": ["post", "special-dispatch"] , "outs": ["final.txt"] } , "": { "type": "export" , "target": "internal" , "flexible_config": ["AR", "ARCH", "ENV", "OS"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.fail000066400000000000000000000002661516554100600316460ustar00rootroot00000000000000{ "": {"type": "export", "target": "FaIlInGtArGeT"} , "FaIlInGtArGeT": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["echo -n FaIl", "echo UrEmEsSaGe", "exit 42"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.slow000066400000000000000000000006561516554100600317220ustar00rootroot00000000000000{ "": {"type": "export", "target": "slow", "flexible_config": ["RANGE"]} , "slow": { "type": "generic" , "arguments_config": ["RANGE"] , "outs": ["out.txt"] , "cmds": [ "touch out.txt" , { "type": "join" , "$1": [ "for entry in " , {"type": "var", "name": "RANGE"} , " ; " , "do echo $entry >> out.txt ; " , "sleep 1 ; " , "done" ] } ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.transform000066400000000000000000000010261516554100600327410ustar00rootroot00000000000000{ "": {"type": "export", "target": "multiple, upper", "flexible_config": ["DATA"]} , "blob": { "type": "file_gen" , "arguments_config": ["DATA"] , "name": "data.orig" , "data": {"type": "var", "name": "DATA"} } , "multiple": { "type": "generic" , "outs": ["data.tripple"] , "cmds": ["cat data.orig data.orig data.orig > data.tripple"] , "deps": ["blob"] } , "multiple, upper": { "type": "generic" , "outs": ["data.txt"] , "cmds": ["cat data.tripple | tr a-z A-Z > data.txt"] , "deps": ["multiple"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/data/targets/TARGETS.tree000066400000000000000000000001631516554100600316660ustar00rootroot00000000000000{ "": {"type": "export", "target": "tree"} , "tree": {"type": "install", "dirs": [[["TREE", null, "src"], "."]]} } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/deduplication_of_serve_requests.sh000066400000000000000000000054041516554100600343200ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOG="${TEST_TMPDIR}/log.txt" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json < TARGETS <<'EOF' { "": {"type": "install", "files": {"a-1": "a-1", "a-2": "a-2", "b": "b"}} , "a-1": { "type": "configure" , "target": ["@", "transform", "", ""] , "config": { "type": "let*" , "bindings": [["DATA", "a"], ["unrelated", "foo"]] , "body": {"type": "env", "vars": ["DATA", "unrelated"]} } } , "a-2": { "type": "configure" , "target": ["@", "transform", "", ""] , "config": { "type": "let*" , "bindings": [["DATA", "a"], ["unrelated", "bar"]] , "body": {"type": "env", "vars": ["DATA", "unrelated"]} } } , "b": { "type": "configure" , "target": ["@", "transform", "", ""] , "config": { "type": "let*" , "bindings": [["DATA", "b"], ["unrelated", "foo"]] , "body": {"type": "env", "vars": ["DATA", "unrelated"]} } } } EOF # As from the 3 absent export targets two coincide on the flexible # variables, we should only get two export targets served. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ -R "${SERVE}" -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} \ build -f "${LOG}" --log-limit 4 2>&1 echo grep 'xport.*2 served' "${LOG}" echo # The same should be true on the second run, when everything is in # the cache of serve. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ -R "${SERVE}" -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} \ build -f "${LOG}" --log-limit 4 2>&1 echo grep 'xport.*2 served' "${LOG}" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/deep-tree.sh000066400000000000000000000052071516554100600275240ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBRA="${TEST_TMPDIR}/local-build-root-a" readonly LBRB="${TEST_TMPDIR}/local-build-root-b" readonly LBRC="${TEST_TMPDIR}/local-build-root-c" readonly OUT="${PWD}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json <&1 # Santity check: we find data2.txt and verify the content find "${OUT}/result" -name data2.txt 2>&1 grep 2 $(find "${OUT}/result" -name data2.txt) # Verify that the deep tree can also be handled by install-cas # with and without the --remember option cat artifacts.json echo "${JUST_MR}" --norc --local-build-root "${LBRB}" \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install-cas --remember -o "${OUT}/first-copy" \ $(jq -r '."'"${OUT}/result/out"'".id' artifacts.json)::t \ 2>&1 "${JUST_MR}" --norc --local-build-root "${LBRC}" \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install-cas -o "${OUT}/second-copy" \ $(jq -r '."'"${OUT}/result/out"'".id' artifacts.json)::t \ 2>&1 # sanity check of the copies grep 2 $(find "${OUT}/first-copy" -name data2.txt) grep 2 $(find "${OUT}/second-copy" -name data2.txt) echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/describe.sh000066400000000000000000000034521516554100600274320ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR1="${TEST_TMPDIR}/local-build-root-1" readonly LBR2="${TEST_TMPDIR}/local-build-root-1" readonly OUT="${PWD}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi echo Local description echo (cd repo \ && "${JUST}" describe --local-build-root "${LBR1}" > "${OUT}/describe.orig") cat "${OUT}/describe.orig" echo mkdir work cd work touch ROOT cat > repos.json < "${OUT}/describe" cat "${OUT}/describe" echo diff -u "${OUT}/describe.orig" "${OUT}/describe" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/failure-report.sh000066400000000000000000000044721516554100600306150ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${PWD}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json <&1 \ && exit 1 || : echo cat "${OUT}/serve.log" echo SERVE_LOG=$(jq -rM '.[0][1]' "${OUT}/serve.log") echo Serve log has blob ${SERVE_LOG} echo # The serve log blob must be mentioned in the log file grep ${SERVE_LOG} "${OUT}/log" echo # Fetch the failure log "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install-cas -o "${OUT}/failure.log" ${SERVE_LOG} 2>&1 echo cat "${OUT}/failure.log" echo # Failure log must contain error message grep FaIlUrEmEsSaGe "${OUT}/failure.log" # Failure log must contain exit code grep 42 "${OUT}/failure.log" # Failure log must contain failing targets grep FaIlInGtArGeT "${OUT}/failure.log" echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/interruption-test.sh000066400000000000000000000067571516554100600314040ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${PWD}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json <&1 & pid="$!" pids="${pids} ${pid}" done echo echo Starting processes expected to be interrupted echo for i in `seq 20 35` do echo Starting build with parameter $i "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -f "${OUT}/proc$i.log" \ --restrict-stderr-log-limit 1 \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ build -D '{"RANGE": "`seq 1 '"${i}"'`"}' \ 2>&1 & pid="$!" pids="${pids} ${pid}" killpids="${killpids} ${pid}" done echo "Started ${pids}" sleep 7 echo "Stopping processes ${killpids}" for pid in ${killpids}; do kill $pid done echo "Waiting for processes ${pids}" for pid in ${pids}; do wait $pid || echo "$pid has exit code $?" done echo done echo # Serve should not be affected by clients disappearing and we # still should be able to build, both targets already requested # as well as new ones. "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -f "${OUT}/finalout3.log" \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT}/finalout3" -D '{"RANGE": "`seq 1 3`"}' 2>&1 echo "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ -f "${OUT}/finalout7.log" \ -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT}/finalout7" -D '{"RANGE": "`seq 1 7`"}' 2>&1 echo echo Sanity checks grep 2 "${OUT}/finalout7/out.txt" grep 6 "${OUT}/finalout7/out.txt" grep 2 "${OUT}/finalout3/out.txt" grep 6 "${OUT}/finalout3/out.txt" && exit 1 || : echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/load-test.sh000066400000000000000000000036641516554100600275530ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${PWD}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json <&1 & pid="$!" pids="${pids} ${pid}" done echo "Waiting for processes ${pids}" for pid in ${pids}; do wait $pid || echo "$pid has exit code $?" done echo done echo # Sanity check: files contain some numbers for i in `seq 1 2` do grep 123 "${OUT}/out${i}/deep" grep 123 "${OUT}/out${i}/wide" done echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root/000077500000000000000000000000001516554100600310365ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root/TARGETS000066400000000000000000000070711516554100600320770ustar00rootroot00000000000000{ "serve-tree-syms (repo)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": ["for i in `seq 1 5` ; do ln -s $i.txt src/link-$i ; done"] , "deps": [["end-to-end/serve-service", "serve-target-remote-build (repo)"]] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "serve-tree-syms (data)": {"type": "install", "dirs": [["serve-tree-syms (repo)", "repo"]]} , "serve-tree-syms (archive)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "outs": ["src.tar"] , "cmds": ["tar cf src.tar repo"] , "deps": ["serve-tree-syms (data)"] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "unresolved-present": { "type": ["end-to-end", "with serve"] , "name": ["unresolved-present"] , "test": ["unresolved-present.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end/serve-service", "serve-tree (archive)"] ] , "repos": [["end-to-end/serve-service", "serve-tree (archive)"]] } , "unresolved-absent": { "type": ["end-to-end", "with serve"] , "name": ["unresolved-absent"] , "test": ["unresolved-absent.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end/serve-service", "serve-tree (archive)"] ] , "repos": [["end-to-end/serve-service", "serve-tree (archive)"]] } , "unresolved-absent-known": { "type": ["end-to-end", "with serve"] , "name": ["unresolved-absent-known"] , "test": ["unresolved-absent-known.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end/serve-service", "serve-tree (archive)"] ] , "repos": [["end-to-end/serve-service", "serve-tree (archive)"]] } , "unresolved-absent-known-upload": { "type": ["end-to-end", "with serve"] , "name": ["unresolved-absent-known-upload"] , "test": ["unresolved-absent-known-upload.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end/serve-service", "serve-tree (archive)"] ] } , "resolved-present": { "type": ["end-to-end", "with serve"] , "name": ["resolved-present"] , "test": ["resolved-present.sh"] , "deps": [ "serve-tree-syms (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-tree-syms (archive)"] } , "resolved-absent": { "type": ["end-to-end", "with serve"] , "name": ["resolved-absent"] , "test": ["resolved-absent.sh"] , "deps": [ "serve-tree-syms (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-tree-syms (archive)"] } , "resolved-absent-known": { "type": ["end-to-end", "with serve"] , "name": ["resolved-absent-known"] , "test": ["resolved-absent-known.sh"] , "deps": [ "serve-tree-syms (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-tree-syms (archive)"] } , "resolved-absent-known-upload": { "type": ["end-to-end", "with serve"] , "name": ["resolved-absent-known-upload"] , "test": ["resolved-absent-known-upload.sh"] , "deps": [ "serve-tree-syms (archive)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["serve-archive-root"] , "deps": [ "resolved-absent" , "resolved-absent-known" , "resolved-absent-known-upload" , "resolved-present" , "unresolved-absent" , "unresolved-absent-known" , "unresolved-absent-known-upload" , "unresolved-present" ] } } resolved-absent-known-upload.sh000066400000000000000000000061601516554100600370270ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint # does not know the archive. # # The test archive contains symlinks to be resolved, which tests also the # resolved tree file association. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup absent) cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK resolved-absent-known.sh000066400000000000000000000057761516554100600355610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint # knows the archive. # # The test archive contains symlinks to be resolved, which tests also the # resolved tree file association. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup absent) cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK resolved-absent.sh000066400000000000000000000063311516554100600344130ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made by asking # the serve endpoint to set it up for us when we have no local knowledge. # # The test archive contains symlinks to be resolved, which tests also the # resolved tree file association. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json < repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # We now test if the serve endpoint can provide us the root. # In a clean build root, ask serve to set up the root for us, from scratch rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # Double-check the file association was created and root remains available # without the remote endpoints ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK unresolved-absent-known-upload.sh000066400000000000000000000057351516554100600374010ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint # does not know the archive. # # The test archive does not contain symlinks. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup absent) cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK unresolved-absent-known.sh000066400000000000000000000055531516554100600361150ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint # knows the archive. # # The test archive does not contain symlinks. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup absent) cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK unresolved-absent.sh000066400000000000000000000061061516554100600347560ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-archive-root#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made by asking # the serve endpoint to set it up for us when we have no local knowledge. # # The test archive does not contain symlinks. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) mkdir work cd work touch ROOT cat > repos.json < repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # We now test if the serve endpoint can provide us the root. # In a clean build root, ask serve to set up the root for us, from scratch rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # Double-check the file association was created and root remains available # without the remote endpoints ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/000077500000000000000000000000001516554100600310575ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/TARGETS000066400000000000000000000030501516554100600321110ustar00rootroot00000000000000{ "serve-distdir (data)": { "type": "install" , "files": { "src.tar": ["end-to-end/serve-service", "serve-tree (archive)"] , "src-syms.tar": [ "end-to-end/serve-service/serve-archive-root" , "serve-tree-syms (archive)" ] , "unrelated.txt": "unrelated file" } } , "unrelated file": {"type": "file_gen", "name": "unrelated", "data": "unrelated content"} , "present": { "type": ["end-to-end", "with serve"] , "name": ["present"] , "test": ["present.sh"] , "deps": [ "serve-distdir (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-distdir (data)"] } , "absent": { "type": ["end-to-end", "with serve"] , "name": ["absent"] , "test": ["absent.sh"] , "deps": [ "serve-distdir (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] , "repos": ["serve-distdir (data)"] } , "upload": { "type": ["end-to-end", "with serve"] , "name": ["upload"] , "test": ["upload.sh"] , "deps": [ "serve-distdir (data)" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] ] } , "foreign-file": { "type": ["end-to-end", "with serve"] , "name": ["foreign-file"] , "test": ["foreign-file.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["TREE", null, "foreign-file-data"] ] , "repos": [["TREE", null, "foreign-file-data"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["serve-distdir-root"] , "deps": ["absent", "foreign-file", "present", "upload"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/absent.sh000066400000000000000000000067221516554100600326760ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent root can successfully be made by asking # the serve endpoint to set it up for us when we have no local knowledge. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" SRC_HASH=$(git hash-object src.tar) cp src-syms.tar "${DISTDIR}" SRC_SYMS_HASH=$(git hash-object src-syms.tar) mkdir work cd work touch ROOT cat > repos.json < out.txt"] , "deps": ["hello.txt"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh000066400000000000000000000044111516554100600337610ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly DATA_DIR="${PWD}/foreign-file-data" readonly OUT="${TEST_TMPDIR}/out" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" echo echo Will use endpoint arguments ${ENDPOINT_ARGS} DATA_HASH=$(git hash-object "${DATA_DIR}/data.txt") TARGET_HASH=$(git hash-object "${DATA_DIR}/the-targets-file") mkdir work cd work touch ROOT cat > repos.json <&1 grep 'HELLO WORLD' "${OUT}/out.txt" # also verify that the repo config has the repository absent CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" ${ENDPOINT_ARGS} setup) echo echo Configuration used was ${CONF} echo cat "${CONF}" echo test $(jq '.repositories.""."workspace_root" | length' "${CONF}") -eq 2 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/present.sh000066400000000000000000000075621516554100600331050ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks 3 of the options to make a present root for a distidr # repository, where: # - archives are in a local distfile; # - there is already a file association to the distdir root tree; # - we receive the distdir root from serve endpoint via the remote CAS. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" SRC_HASH=$(git hash-object src.tar) cp src-syms.tar "${DISTDIR}" SRC_SYMS_HASH=$(git hash-object src-syms.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # We now test if the serve endpoint can provide us the present root. # In a clean build root, ask serve to set up the root for us, from scratch rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" # Double-check the file association was created and root remains available # without the remote endpoints ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-distdir-root/upload.sh000066400000000000000000000063301516554100600327010ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that an absent distdir root can be successfully computed # locally and then uploaded to a serve endpoint that does not know the root. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" mkdir -p "${DISTDIR}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## cp src.tar "${DISTDIR}" SRC_HASH=$(git hash-object src.tar) cp src-syms.tar "${DISTDIR}" SRC_SYMS_HASH=$(git hash-object src-syms.tar) mkdir work cd work touch ROOT cat > repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ ${ENDPOINT_ARGS} setup absent) cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-file-root/000077500000000000000000000000001516554100600303345ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-file-root/TARGETS000066400000000000000000000005531516554100600313730ustar00rootroot00000000000000{ "upload": { "type": ["end-to-end", "with serve"] , "name": ["upload"] , "test": ["upload.sh"] , "deps": [ ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["end-to-end/serve-service", "serve-tree (archive)"] ] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["serve-file-root"] , "deps": ["upload"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-file-root/upload.sh000066400000000000000000000054421516554100600321610ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that the root for a file repository marked to_git is being # uploaded to the serve endpoint. # # Also checks the upload of locally known absent roots for Git-tree # repositories. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LOCAL_REPO="${TEST_TMPDIR}/repo_root" readonly LBR="${TEST_TMPDIR}/local-build-root" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}" ### # Setup sample repos config with present and absent repos ## mkdir -p "${LOCAL_REPO}" tar xf src.tar -C "${LOCAL_REPO}" cd "${LOCAL_REPO}" git init -b trunk git config user.name 'N.O.Body' git config user.email 'nobody@example.org' git add -f . git commit -m 'Test repo' 2>&1 TREE=$(git log -n 1 --format="%T") cd - mkdir work cd work touch ROOT cat > repos.json <&1 COMMIT=$(git log -n 1 --format="%H") SUBTREE=$(git ls-tree "$COMMIT" repo | awk '{print $3}') cd - mkdir work cd work touch ROOT cat > repos.json < repos.json < repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" # Check that the subdir is also working correctly ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 cat > repos.json < repos.json < repos.json <&1 ${JUST} gc --local-build-root ${LBR} 2>&1 CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ --log-limit 6 \ setup main) cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve-tree.sh000077500000000000000000000055761516554100600277470ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR_DEBUG="${TEST_TMPDIR}/build-root-DEBUG" readonly LBR_A="${TEST_TMPDIR}/build-root-A" readonly LBR_B="${TEST_TMPDIR}/build-root-B" readonly LBR_C="${TEST_TMPDIR}/build-root-C" readonly LBR_D="${TEST_TMPDIR}/build-root-D" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly TARGET_ROOT="${PWD}/data/targets" mkdir -p "${DISTDIR}" cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi REMOTE="-r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT}" mkdir work cd work touch ROOT cat > repos.json <&1 echo echo Remote build "${JUST_MR}" --norc --local-build-root "${LBR_B}" --just "${JUST}" \ --distdir "${DISTDIR}" ${REMOTE} build \ --log-limit 4 \ --dump-artifacts remote.json 2>&1 echo echo Serve build "${JUST_MR}" --norc --local-build-root "${LBR_C}" --just "${JUST}" \ --distdir "${DISTDIR}" ${REMOTE} -R ${SERVE} build \ --log-limit 4 \ --dump-artifacts serve.json 2>&1 echo echo Absent build echo -n '[""]' > abs "${JUST_MR}" --norc --local-build-root "${LBR_D}" --just "${JUST}" \ --distdir "${DISTDIR}" ${REMOTE} -R ${SERVE} --absent abs build \ --log-limit 4 \ --dump-artifacts absent.json 2>&1 diff -u local.json remote.json diff -u local.json serve.json diff -u local.json absent.json echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_export_deps.sh000066400000000000000000000106251516554100600314120ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # We test that we are able to compile an export target, which depends on an # absent one. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LOCAL_DIR="${TEST_TMPDIR}/local" readonly ABSENT_DIR="${TEST_TMPDIR}/absent" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi # Set up sample repository readonly GENERATOR="${TEST_TMPDIR}/generate.sh" readonly GEN_DIR="{TEST_TMPDIR}/gen-dir" cat > "${GENERATOR}" < TARGETS < out.txt"] , "outs": ["out.txt"] } , "main": {"type": "export", "target": "main-internal", "flexible_config": ["ENV"]} } ENDTARGETS EOF chmod 755 "${GENERATOR}" mkdir -p "${GEN_DIR}" ( cd "${GEN_DIR}" git init git config user.email "nobody@example.org" git config user.name "Nobody" "${GENERATOR}" git add . git commit -m "first commit" ) readonly TREE_ID=$(cd "${GEN_DIR}" && git log -n 1 --format="%T") # fill the target cache that will be used by just serve mkdir -p ${LOCAL_DIR} ( cd ${LOCAL_DIR} touch ROOT cat > repos.json < "${GENERATOR_LOCAL}" < TARGETS < local.txt"] , "outs": ["local.txt"] , "deps" : [["@", "absent-dep", "", "main"]] } , "main": {"type": "export", "target": "main-internal", "flexible_config": ["ENV"]} } EOFTARGETS EOF chmod 755 "${GENERATOR_LOCAL}" mkdir -p "${GEN_DIR_LOCAL}" ( cd "${GEN_DIR_LOCAL}" git init git config user.email "nobody@example.org" git config user.name "Nobody" "${GENERATOR_LOCAL}" git add . git commit -m "first commit" ) readonly TREE_ID_LOCAL=$(cd "${GEN_DIR_LOCAL}" && git log -n 1 --format="%T") # test with the absent repository mkdir -p "${ABSENT_DIR}" ( cd "${ABSENT_DIR}" touch ROOT cat > repos.json < repos.json < TARGETS <&1 [ "$(cat "${OUT}/a")" = "AAA" ] [ "$(cat "${OUT}/b")" = "BBB" ] [ "$(cat "${OUT}/c")" = "CCC" ] [ "$(cat "${OUT}/d")" = "DDD" ] [ "$(cat "${OUT}/e")" = "EEE" ] [ "$(cat "${OUT}/f")" = "FFF" ] done echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_query_target_cache_value.sh000066400000000000000000000135761516554100600341200ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # Check that a build will attempt to get the target cache value from just serve # during analysis of an export target in the case of a local target cache miss. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly LBR_UNRELATED_A="${TEST_TMPDIR}/local-build-root-unrelated-A" readonly LBR_UNRELATED_B="${TEST_TMPDIR}/local-build-root-unrelated-B" readonly TOOLS_DIR="${TEST_TMPDIR}/tools" readonly OUT="${TEST_TMPDIR}/out" readonly DISPATCH_FILE="${TEST_TMPDIR}/dispatch.json" cat > "${DISPATCH_FILE}" < "${TOOLS_DIR}/tree" <<'EOF' #!/bin/sh mkdir -p $1/hello/world/tree echo Hello World > $1/hello/world/tree/hello.txt echo -n World > $1/hello/world/tree/name.txt EOF chmod 755 "${TOOLS_DIR}/tree" mkdir work cd work touch WORKSPACE cat > TARGETS <<'EOF' { "tree": { "type": "generic" , "arguments_config": ["ENV"] , "out_dirs": ["out"] , "cmds": ["${TOOLS}/tree out"] , "env": {"type": "var", "name": "ENV"} } , "": { "type": "export" , "flexible_config": ["ENV"] , "target": "tree" } } EOF cat > repos.json <<'EOF' { "main": "" , "repositories": { "": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } EOF cat repos.json cat TARGETS # Build to fill the target cache of the serve endpoint CONF=$("${JUST_MR}" --norc --local-build-root "${SERVE_LBR}" setup) echo "generated conf": cat "${CONF}" echo "${JUST}" build \ --local-build-root "${LBR_UNRELATED_A}" \ -C "${CONF}" \ --remote-serve-address "${SERVE}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # Demonstrate that from now on, we don't build anything any more rm -rf "${TOOLS_DIR}" # Setup for a build in a new build root CONF=$("${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" setup) echo "generated conf": cat "${CONF}" echo # Demonstrate that we can analyse, but not build locally "${JUST}" analyse \ --local-build-root "${LBR}" \ -C "${CONF}" \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 "${JUST}" build \ --local-build-root "${LBR}" \ -C "${CONF}" \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 && echo "this should fail" && exit 1 echo "failed as expected" # Demonstrate we cannot build with a clean remote CAS "${JUST}" gc --local-build-root ${REMOTE_LBR} 2>&1 if [ -n "${STANDALONE_SERVE:-}" ] then # if serve and remote-execution are the same process # they also use the same local build root; in this case, # we need to keep the target-level cache of the serve # process alive echo "Building with serve again, to keep tc cache alive" "${JUST}" build \ --local-build-root "${LBR_UNRELATED_B}" \ -C "${CONF}" \ --remote-serve-address "${SERVE}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 fi "${JUST}" gc --local-build-root ${REMOTE_LBR} 2>&1 echo "${JUST}" build \ --local-build-root "${LBR}" \ -C "${CONF}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 && echo "this should fail" && exit 1 echo "failed as expected" # Demonstrate that we can build if serve endpoint provides the target cache value "${JUST}" build \ --local-build-root "${LBR}" \ -C "${CONF}" \ --remote-serve-address "${SERVE}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ --log-limit 5 \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 # Verify that the export target is fully in cache "${JUST}" install \ --local-build-root "${LBR}" \ -C "${CONF}" \ --remote-serve-address "${SERVE}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ --log-limit 5 \ -o "${OUT}" \ -D '{"ENV": {"TOOLS": "'${TOOLS_DIR}'"}}' 2>&1 ls -R "${OUT}" test -f "${OUT}/out/hello/world/tree/hello.txt" test -f "${OUT}/out/hello/world/tree/name.txt" test "$(cat "${OUT}/out/hello/world/tree/name.txt")" = "World" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_start_execute.sh000066400000000000000000000050541516554100600317350ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # By design, when a just-serve instance is created, if no remote-execution # endpoint is provided, the same process will act also as just-execute. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly INFOFILE="${PWD}/info.json" readonly PIDFILE="${PWD}/pid.txt" # test that, if no remote endpoint is passed to just-serve, it will spawn a # just-execute instance COMPAT="" cat > .just-servec <> .just-servec <> .just-servec <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") cleanup() { kill $(cat "${PIDFILE}") } trap cleanup EXIT touch ROOT cat > TARGETS < out.txt"] , "outs": ["out.txt"] } } ENDTARGETS "${JUST}" install --local-build-root "${LBR}" \ -r localhost:${PORT} ${COMPAT} -o . grep 'just-serve-just-execute' out.txt # test that if we only pass --remote-serve-address it is also used as remote # execution endpoint rm -rf "${LBR}" "${JUST}" install --local-build-root "${LBR}" \ --remote-serve-address localhost:${PORT} ${COMPAT} -o . grep 'just-serve-just-execute' out.txt just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_start_execute_sharding.sh000066400000000000000000000107541516554100600336170ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # In this script, we first fill the target cache of a just-serve instance # by building an export target. Then, we check that just-serve can get a # cache hit if set up to act also as just-execute. # # The remote properties and dispatch file are used to test that both # client and server shard the target cache entry in the same way. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly SERVE_LBR="${TEST_TMPDIR}/serve-local-build-root" readonly LOCAL_DIR="${TEST_TMPDIR}/local" readonly ABSENT_DIR="${TEST_TMPDIR}/absent" readonly INFOFILE="${PWD}/info.json" readonly PIDFILE="${PWD}/pid.txt" ## # Set up sample repository # readonly GENERATOR="${TEST_TMPDIR}/generate.sh" readonly GEN_DIR="{TEST_TMPDIR}/gen-dir" cat > "${GENERATOR}" < TARGETS < out.txt"] , "outs": ["out.txt"] } , "main": {"type": "export", "target": "main-internal", "flexible_config": ["ENV"]} } ENDTARGETS EOF cat "${GENERATOR}" chmod 755 "${GENERATOR}" mkdir -p "${GEN_DIR}" ( cd "${GEN_DIR}" git init git config user.email "nobody@example.org" git config user.name "Nobody" "${GENERATOR}" git add . git commit -m "first commit" ) readonly TREE_ID=$(cd "${GEN_DIR}" && git log -n 1 --format="%T") ## # Fill the target cache that will be used by just serve by building locally # mkdir -p ${LOCAL_DIR} ( cd ${LOCAL_DIR} touch ROOT cat > repos.json < .just-servec <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") cleanup() { kill $(cat "${PIDFILE}") } ## # Change repository to absent and unbuildable (remove generator script) and # test that we can build from the populated build root, as we should get a # target cache hit, assuming sharding is done as for a local build # mkdir -p "${ABSENT_DIR}" ( cd "${ABSENT_DIR}" touch ROOT cat > repos.json < "${DISPATCH_FILE}" < "${GENERATOR}" < TARGETS < out.txt"] , "outs": ["out.txt"] } , "main": {"type": "export", "target": "main-internal", "flexible_config": ["ENV"]} } ENDTARGETS EOF cat "${GENERATOR}" chmod 755 "${GENERATOR}" mkdir -p "${GEN_DIR}" ( cd "${GEN_DIR}" git init git config user.email "nobody@example.org" git config user.name "Nobody" "${GENERATOR}" git add . git commit -m "first commit" ) readonly TREE_ID=$(cd "${GEN_DIR}" && git log -n 1 --format="%T") # fill the target cache that will be used by just serve mkdir -p ${LOCAL_DIR} ( cd ${LOCAL_DIR} touch ROOT cat > repos.json < repos.json <&2 && exit 1 echo "failed as expected" # test that we can successfully compile using just serve "${JUST}" build \ --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address "${SERVE}" \ -C "${CONF}" \ -r "${REMOTE_EXECUTION_ADDRESS}" \ ${COMPAT} \ ${REMOTE_PROPERTIES} \ ${DISPATCH} \ main ) just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_target_failed_build.sh000066400000000000000000000104571516554100600330320ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # This script aims to test the "remote build capabilities" of a just-serve # instance. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR_1="${TEST_TMPDIR}/local-build-root-1" readonly LBR_2="${TEST_TMPDIR}/local-build-root-2" readonly LBR_3="${TEST_TMPDIR}/local-build-root-3" readonly OUTPUT_1="${TEST_TMPDIR}/output-dir-1" readonly OUTPUT_2="${TEST_TMPDIR}/output-dir-2" readonly DISTDIR="${TEST_TMPDIR}/distfiles" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir -p "${DISTDIR}" cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) readonly TARGETS_ROOT="${PWD}/data/targets" readonly RULES_ROOT="${PWD}/data/rules" mkdir work cd work touch ROOT cat > repos.json <&1 for i in $(seq 5); do grep "./tree/src/$i.txt" ${OUTPUT_1}/_out done echo # Check that build succeeds with a serve endpoint present # # Reason: serve endpoint does not have the correct targets and rules root and # thus fails, but locally we can continue. ${JUST} install --local-build-root "${LBR_2}" -C "${CONF}" \ --log-limit 5 \ --remote-serve-address "${SERVE}" \ -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} \ -o "${OUTPUT_2}" 2>&1 for i in $(seq 5); do grep "./tree/src/$i.txt" ${OUTPUT_2}/_out done echo # Check that build fails with a serve endpoint present if the orchestrated build # actually fails # # Reason: the serve endpoint has all the roots to start the analysis/build of # the target, but we use a dummy rule and thus build fails. This is reported to # the client and in this case the whole build is expected to fail. rm "${RULES_ROOT}/RULES" # to match known rule tree on serve endpoint cat > repos.json <&1 && exit 1 || : echo Failed as expected echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/serve_target_remote_build.sh000066400000000000000000000051731516554100600331000ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # This script aims to test the "remote build capabilities" of a just-serve # instance. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUTPUT="${TEST_TMPDIR}/output-dir" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi mkdir work cd work touch ROOT cat > repos.json <&1 for i in $(seq 5); do grep "./tree/src/$i.txt" ${OUTPUT}/_out done echo OK serve_target_remote_build_dispatch.sh000066400000000000000000000072701516554100600347000ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ########################################################################### # # This script aims to test the correct dispatch of a build executed by the # just serve endpoint. # ########################################################################### set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBRDIR="${TEST_TMPDIR}/local-build-root" readonly ESDIR="${TEST_TMPDIR}/dispatch-build-root" readonly OUTPUT="${TEST_TMPDIR}/output-dir" readonly INFOFILE="${PWD}/info.json" readonly PIDFILE="${PWD}/pid.txt" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi # To test the dispatch, we have a remote-execution server # with a special variant of cat that # - prefixes with some extra string, but # - only takes the first argument into account. # In this way, we can distinguish it from the regular cat on the # on the normal remote execution server. readonly REFERENCE_OUTPUT="FooOOOooo" readonly SERVER_BIN_DIR="${TMPDIR}/server/bin" mkdir -p "${SERVER_BIN_DIR}" cat > "${SERVER_BIN_DIR}/cat" <&1 & for _ in `seq 1 60` do if test -f "${INFOFILE}" then break fi sleep 1; done if ! test -f "${INFOFILE}" then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") FAILED="" # set up the absent targets to build mkdir work cd work touch ROOT cat > dispatch.json < repos.json <&1 || FAILED=YES kill $(cat "${PIDFILE}") [ -z "${FAILED}" ] [ -f "${OUTPUT}/final.txt" ] echo echo content of final.txt cat "${OUTPUT}/final.txt" echo grep ${REFERENCE_OUTPUT} "${OUTPUT}/final.txt" grep this-is-the-payload "${OUTPUT}/final.txt" grep drop "${OUTPUT}/final.txt" && exit 1 || : echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/serve-service/wide-deep-targets.py000066400000000000000000000040631516554100600312010ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from typing import Any, Dict, cast SIZE=1000 deep : Dict[str, Any] = ( {"deep%d (unexported)" % (i,) : {"type": "generic", "deps": ["deep%d" % (i-1,)] if i > 0 else [], "outs": ["out%d" % (i,)], "cmds": ["cat out%d > out%d" % (i-1, i) if i > 0 else "true", "echo %d >> out%d" % (i, i)]} for i in range(SIZE)} | {"deep%d" % (i,) : {"type": "export", "target": "deep%d (unexported)" %(i,)} for i in range(SIZE)} ) wide : Dict[str, Any] = ( cast(Dict[str, Any], {"wide": {"type": "generic", "outs": ["out"], "cmds": ["for f in $(ls wide* | sort) ; do cat $f >> out ; done"], "deps": ["wide%d" % (i,) for i in range(SIZE)] }}) | cast(Dict[str, Any], {"wide%d (unexported)" % (i,) : {"type": "generic", "outs": ["wide%d" % (i,)], "cmds": ["echo %d > wide%d" % (i,i)]} for i in range(SIZE)}) | cast(Dict[str, Any], {"wide%d" % (i,) : {"type": "export", "target": "wide%d (unexported)" %(i,)} for i in range(SIZE)}) ) total : Dict[str, Any] = ( { "": {"type": "export", "target": "all"}, "all": {"type": "install", "files": {"deep": "deep%d" % (SIZE-1,), "wide": "wide"}}} | deep | wide) with open("TARGETS", "w") as f: json.dump(total, f) just-buildsystem-justbuild-b1fb5fa/test/end-to-end/symlinks/000077500000000000000000000000001516554100600244015ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/symlinks/TARGETS000066400000000000000000000007521516554100600254410ustar00rootroot00000000000000{ "stage-links": { "type": ["end-to-end", "with remote"] , "name": ["stage-links"] , "test": ["stage-links.sh"] , "deps": [["", "tool-under-test"]] } , "tree-reference": { "type": ["end-to-end", "with remote"] , "name": ["tree-reference"] , "test": ["tree-reference.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["symlinks"] , "deps": ["stage-links", "tree-reference"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/symlinks/stage-links.sh000066400000000000000000000105771516554100600271700ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly OUT="${TMPDIR}/out" mkdir -p "${OUT}" touch ROOT cat > TARGETS <&1 ${JUST} install -L '["env", "PATH='"${PATH}"'"]' ${ARGS} input-non-upwards \ -o "${OUT}" 2>&1 && ls -alR "${OUT}" && rm -rf "${OUT}/*" || FAILED=YES echo echo "test input non-upwards remotely" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} input-non-upwards 2>&1 ${JUST} install -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} input-non-upwards \ -o "${OUT}" 2>&1 && ls -alR "${OUT}" && rm -rf "${OUT}/*" || FAILED=YES echo echo "test staging non-upwards locally" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} stage-non-upwards-links 2>&1 || FAILED=YES echo echo "test staging non-upwards remotely" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} stage-non-upwards-links 2>&1 || FAILED=YES # Check that actions with non-contained upwards symlinks fail echo echo "test input non-contained locally" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} input-non-contained 2>&1 \ && echo "this should have failed" && FAILED=YES ${JUST} install -L '["env", "PATH='"${PATH}"'"]' ${ARGS} input-non-contained -o "${OUT}" 2>&1 \ && echo "this should have failed" && FAILED=YES \ && ls -alR "${OUT}" && rm -rf "${OUT}/*" || echo "failed as expected" echo echo "test input non-contained remotely" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} input-non-contained 2>&1 \ && echo "this should have failed" && FAILED=YES || echo "failed as expected" ${JUST} install -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} input-non-contained -o "${OUT}" 2>&1 \ && echo "this should have failed" && FAILED=YES \ && ls -alR "${OUT}" && rm -rf "${OUT}/*" || echo "failed as expected" echo echo "test staging non-contained locally" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} stage-non-contained-links 2>&1 \ && echo "this should have failed" && FAILED=YES || echo "failed as expected" echo echo "test staging non-contained remotely" ${JUST} build -L '["env", "PATH='"${PATH}"'"]' ${ARGS} ${REMOTE_EXECUTION_ARGS} stage-non-contained-links 2>&1 \ && echo "this should have failed" && FAILED=YES || echo "failed as expected" if [ ! -z "${FAILED}" ]; then exit 1 fi echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/symlinks/tree-reference.sh000066400000000000000000000043001516554100600276250ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that roots with upwards symlinks can be added to the Git CAS, # but referencing invalid entries fails in analysis. ## set -eu env readonly ROOT="$(pwd)" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" readonly OUT="${TEST_TMPDIR}/out" readonly SRCDIR="${ROOT}/src" readonly WRKDIR="${ROOT}/work" # Set up data with a valid symlink mkdir -p "${SRCDIR}/data-subtree" "${SRCDIR}/links-subtree" touch "${SRCDIR}/data-subtree/something" ln -s NON_EXISTENT "${SRCDIR}/links-subtree/valid" # valid symlink cat > "${SRCDIR}/TARGETS" << EOF { "": { "type": "generic" , "outs": ["out.txt"] , "cmds": ["ls -alR > out.txt"] , "deps": [["TREE", null, "data-subtree"], ["TREE", null, "links-subtree"]] } } EOF # Main repo depending on repo 'bar' mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT cat > repos.json << EOF { "repositories": { "": { "repository": {"type": "file", "path": "${SRCDIR}", "pragma": {"to_git": true}} } } } EOF # Test success with valid symlink in links-subtree "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' install -o ${OUT} 2>&1 echo cat "${OUT}/out.txt" echo # Test analysis failure if invalid (upwards) symlink is added to links-subtree ln -s ../NON_EXISTENT "${SRCDIR}/links-subtree/invalid" "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' analyse 2>&1 \ && echo "this should fail" && exit 1 || : echo echo "failed as expected" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/000077500000000000000000000000001516554100600250575ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/TARGETS000066400000000000000000000034221516554100600261140ustar00rootroot00000000000000{ "target-cache-hit": { "type": ["end-to-end", "with remote"] , "name": ["target-cache-hit"] , "test": ["target-cache-hit.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "artifacts-sync": { "type": ["@", "rules", "shell/test", "script"] , "name": ["artifacts-sync"] , "test": ["artifacts-sync.sh"] , "deps": [ "bootstrap-src-staged" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["./", "data", "greetlib"] , ["./", "data", "pydicts"] ] } , "serve-sync": { "type": ["end-to-end", "with serve"] , "name": ["serve-sync"] , "test": ["serve-sync.sh"] , "deps": [ "bootstrap-src-staged" , ["", "mr-tool-under-test"] , ["", "tool-under-test"] , ["./", "data", "lib with generated hdr"] ] , "repos": [["./", "data", "lib with generated hdr"]] } , "export-extern": { "type": ["@", "rules", "shell/test", "script"] , "name": ["export-extern"] , "test": ["export-extern.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "bootstrap-src-staged": { "type": "install" , "tainted": ["test"] , "dirs": [[["@", "src", "", "bootstrap-src"], "src"]] } , "check-sharding": { "type": ["@", "rules", "shell/test", "script"] , "name": ["check-sharding"] , "test": ["check-sharding.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["target-cache"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "deps": { "type": "++" , "$1": [ ["target-cache-hit"] , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] , "else": ["artifacts-sync", "serve-sync", "check-sharding"] } ] } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/artifacts-sync.sh000066400000000000000000000107161516554100600303520ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly RULES_DIR="${PWD}/src/rules" readonly LOCAL_CACHE="${TMPDIR}/local_cache" readonly REMOTE_CACHE="${TMPDIR}/remote_cache" readonly INFOFILE="${TMPDIR}/info.json" readonly PIDFILE="${TMPDIR}/pid.txt" if [ "${COMPATIBLE:-}" = "YES" ]; then ARGS="--compatible" HASH_TYPE="compatible-sha256" TREE_TAG="f" else ARGS="" HASH_TYPE="git-sha1" TREE_TAG="t" fi readonly FIRST_GEN="${LOCAL_CACHE}/protocol-dependent/generation-0/$HASH_TYPE" readonly TCDIR="$FIRST_GEN/tc" # ------------------------------------------------------------------------------ # Start local remote execution server # ------------------------------------------------------------------------------ "${JUST}" execute --info-file "${INFOFILE}" --pid-file "${PIDFILE}" \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${REMOTE_CACHE}" ${ARGS} 2>&1 & for _ in `seq 1 60`; do if test -f "${INFOFILE}"; then break fi sleep 1; done if ! test -f "${INFOFILE}"; then echo "Did not find ${INFOFILE}" exit 1 fi readonly PORT=$(jq '."port"' "${INFOFILE}") cleanup() { kill $(cat "${PIDFILE}") } trap cleanup EXIT check_main_blobs() { for entry in "${TCDIR}"/* do for sbdir in "${entry}"/* do for f in "${sbdir}"/* do hash=$(cat $f) TC_ENTRY=$("$JUST" install-cas --local-build-root "${LOCAL_CACHE}" $ARGS ${hash}) FILE_HASH=${FILE_HASH:-$(echo $TC_ENTRY | jq -r '.artifacts."libgreet.a".data.id // ""')} TREE_HASH=${TREE_HASH:-$(echo $TC_ENTRY | jq -r '.runfiles.greet.data.id // ""')} "$JUST" install-cas --local-build-root "${LOCAL_CACHE}" ${ARGS} ${FILE_HASH}::f > /dev/null "$JUST" install-cas --local-build-root "${LOCAL_CACHE}" ${ARGS} ${TREE_HASH}::${TREE_TAG} > /dev/null done done done } # ------------------------------------------------------------------------------ # Test synchronization of artifacts in the 'artifacts' and 'runfiles' maps # ------------------------------------------------------------------------------ cd greetlib sed -i "s||${RULES_DIR}|" repos.json # Build greetlib remotely "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ -L '["env", "PATH='"${PATH}"'"]' \ build ${ARGS} -r localhost:${PORT} --dump-graph graph.json main 2>&1 # Count actions without tc EXPECTED=4 readonly ACTIONS_NO_TC=$(cat graph.json | jq '.actions | length' ) test ${ACTIONS_NO_TC} -eq ${EXPECTED} || printf "Wrong number of actions. %d were expected but found %d\n" ${EXPECTED} ${ACTIONS_NO_TC} > /dev/stderr # Check the existence of target-cache dir test -d ${TCDIR} check_main_blobs # Clear remote cache rm -rf "${REMOTE_CACHE}" # Build greetlib remotely "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ -L '["env", "PATH='"${PATH}"'"]' \ build ${ARGS} -r localhost:${PORT} --dump-graph graph-tc.json main 2>&1 # Count actions with tc readonly ACTIONS_TC=$(cat graph-tc.json | jq '.actions | length' ) EXPECTED=2 test ${ACTIONS_TC} -eq ${EXPECTED} || printf "Wrong number of actions. %d were expected but found %d\n" ${EXPECTED} ${ACTIONS_TC} > /dev/stderr # ------------------------------------------------------------------------------ # Test synchronization of artifacts in the 'provides' map # ------------------------------------------------------------------------------ cd ../pydicts rm -rf "${REMOTE_CACHE}" # Build pydicts remotely "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ -L '["env", "PATH='"${PATH}"'"]' \ build ${ARGS} -r localhost:${PORT} json_from_py 2>&1 # Clear remote cache rm -rf "${REMOTE_CACHE}" # Build pydicts remotely "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ -L '["env", "PATH='"${PATH}"'"]' \ build ${ARGS} -r localhost:${PORT} json_from_py 2>&1 just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/check-sharding.sh000066400000000000000000000225621516554100600302740ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ### # This test checks that we correctly shard the target cache. We do this by # building the same generic target in execution endpoint-specific environments # via the launcher argument and checking that we get the expected caching of # the target and the correct action output. ## set -eu env readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR="${TEST_TMPDIR}/lbr" readonly WRKDIR="${PWD}/work" # local paths readonly LOCAL_ROOT="${TEST_TMPDIR}/local" readonly LOCAL_BIN_DIR="${LOCAL_ROOT}/bin" # remote paths readonly REMOTE_ROOT="${TEST_TMPDIR}/remote" readonly REMOTE_CACHE="${REMOTE_ROOT}/build-root" readonly REMOTE_INFOFILE="${REMOTE_ROOT}/info.json" readonly REMOTE_PIDFILE="${REMOTE_ROOT}/pid.txt" readonly REMOTE_BIN_DIR="${REMOTE_ROOT}/bin" # serve standalone paths readonly SERVE_ROOT="${TEST_TMPDIR}/serve" readonly SERVE_CACHE="${SERVE_ROOT}/build-root" readonly SERVE_INFOFILE="${SERVE_ROOT}/info.json" readonly SERVE_PIDFILE="${SERVE_ROOT}/pid.txt" readonly SERVE_BIN_DIR="${SERVE_ROOT}/bin" # serve using remote paths readonly SERVE_RE_ROOT="${TEST_TMPDIR}/serve-remote" readonly SERVE_RE_CACHE="${SERVE_RE_ROOT}/build-root" readonly SERVE_RE_INFOFILE="${SERVE_RE_ROOT}/info.json" readonly SERVE_RE_PIDFILE="${SERVE_RE_ROOT}/pid.txt" readonly SERVE_RE_BIN_DIR="${SERVE_RE_ROOT}/bin" if [ "${COMPATIBLE:-}" = "YES" ]; then ARGS="--compatible" SERVE_COMPAT="true" else ARGS="" fi ## # Set up just execute endpoint with custom launcher # mkdir -p "${REMOTE_BIN_DIR}" cat > "${REMOTE_BIN_DIR}/endpoint" <<'EOF' #!/bin/sh echo remote > $1 EOF chmod 755 "${REMOTE_BIN_DIR}/endpoint" "${JUST}" execute --info-file "${REMOTE_INFOFILE}" --pid-file "${REMOTE_PIDFILE}" \ --local-build-root "${REMOTE_CACHE}" \ -L '["env", "PATH='"${REMOTE_BIN_DIR}:${PATH}"'"]' \ ${ARGS} 2>&1 & for _ in `seq 1 60`; do if test -f "${REMOTE_INFOFILE}"; then break fi sleep 1; done if ! test -f "${REMOTE_INFOFILE}"; then echo "Did not find ${REMOTE_INFOFILE}" exit 1 fi REMOTE_ADDRESS="127.0.0.1:$(jq '."port"' "${REMOTE_INFOFILE}")" ## # Set up standalone just serve endpoint with custom launcher # mkdir -p "${SERVE_BIN_DIR}" cat > "${SERVE_BIN_DIR}/endpoint" <<'EOF' #!/bin/sh echo serve > $1 EOF chmod 755 "${SERVE_BIN_DIR}/endpoint" cat > .just-servec <> .just-servec <> .just-servec <&1 & for _ in `seq 1 60` do if test -f "${SERVE_INFOFILE}" then break fi sleep 1; done if ! test -f "${SERVE_INFOFILE}" then echo "Did not find ${SERVE_INFOFILE}" exit 1 fi SERVE_ADDRESS="127.0.0.1:$(jq '."port"' "${SERVE_INFOFILE}")" ## # Set up just serve endpoint with custom launcher that dispatches to remote # mkdir -p "${SERVE_RE_BIN_DIR}" cat > "${SERVE_RE_BIN_DIR}/endpoint" <<'EOF' #!/bin/sh echo serve-remote > $1 EOF chmod 755 "${SERVE_RE_BIN_DIR}/endpoint" cat > .just-servec <> .just-servec <> .just-servec <&1 & for _ in `seq 1 60` do if test -f "${SERVE_RE_INFOFILE}" then break fi sleep 1; done if ! test -f "${SERVE_RE_INFOFILE}" then echo "Did not find ${SERVE_RE_INFOFILE}" exit 1 fi SERVE_RE_ADDRESS="127.0.0.1:$(jq '."port"' "${SERVE_RE_INFOFILE}")" ## # Set up remotes cleanup # cleanup() { kill $(cat "${REMOTE_PIDFILE}") kill $(cat "${SERVE_PIDFILE}") kill $(cat "${SERVE_RE_PIDFILE}") } trap cleanup EXIT ## # Set up local launcher # mkdir -p "${LOCAL_BIN_DIR}" cat > "${LOCAL_BIN_DIR}/endpoint" <<'EOF' #!/bin/sh echo local > $1 EOF chmod 755 "${LOCAL_BIN_DIR}/endpoint" LOCAL_LAUNCHER='["env", "PATH='"${LOCAL_BIN_DIR}:${PATH}"'"]' ## # Set up the test target to build. Each execution endpoint will know its own # version of the test script to run. # mkdir -p "${WRKDIR}/src" cd "${WRKDIR}" touch ROOT cat > repos.json < src/TARGETS <&1 if ! grep local result/test.out; then echo 'Expected "local" result but found "'$(cat result/test.out)'"' exit 1 fi echo # A regular remote endpoint has some different local TC shard client-side than # a local build. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -r ${REMOTE_ADDRESS} install -o result 2>&1 if ! grep remote result/test.out; then echo 'Expected "remote" result but found "'$(cat result/test.out)'"' exit 1 fi echo # A serve endpoint that dispatches to the same remote as before will have the # same client-side TC shard as if it the build was directly dispatched to said # remote. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -R ${SERVE_RE_ADDRESS} -r ${REMOTE_ADDRESS} install -o result 2>&1 if ! grep remote result/test.out; then echo 'Expected "remote" result but found "'$(cat result/test.out)'"' exit 1 fi echo # Serve standalone only shards like a local build on the serve-side. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -R ${SERVE_ADDRESS} install -o result 2>&1 if ! grep serve result/test.out; then echo 'Expected "serve" result but found "'$(cat result/test.out)'"' exit 1 fi echo # Serve standalone can also be used purely as a remote execution endpoint. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -r ${SERVE_ADDRESS} install -o result 2>&1 if ! grep serve result/test.out; then echo 'Expected "serve" result but found "'$(cat result/test.out)'"' exit 1 fi echo # Check that rebuilding locally after all these runs picks up the correct TC # cache hit. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -L "${LOCAL_LAUNCHER}" install -o result 2>&1 if ! grep local result/test.out; then echo 'Expected "local" result but found "'$(cat result/test.out)'"' exit 1 fi echo # Check that removing all the binaries results in correct cache hits. rm "${LOCAL_BIN_DIR}/endpoint" rm "${REMOTE_BIN_DIR}/endpoint" rm "${SERVE_BIN_DIR}/endpoint" rm "${SERVE_RE_BIN_DIR}/endpoint" "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -L "${LOCAL_LAUNCHER}" install -o result 2>&1 if ! grep local result/test.out; then echo 'Expected "local" result but found "'$(cat result/test.out)'"' exit 1 fi echo "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -r ${REMOTE_ADDRESS} install -o result 2>&1 if ! grep remote result/test.out; then echo 'Expected "remote" result but found "'$(cat result/test.out)'"' exit 1 fi grep remote result/test.out echo "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -R ${SERVE_RE_ADDRESS} -r ${REMOTE_ADDRESS} install -o result 2>&1 if ! grep remote result/test.out; then echo 'Expected "remote" result but found "'$(cat result/test.out)'"' exit 1 fi echo "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -R ${SERVE_ADDRESS} install -o result 2>&1 if ! grep serve result/test.out; then echo 'Expected "serve" result but found "'$(cat result/test.out)'"' exit 1 fi echo "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" \ ${ARGS} -r ${SERVE_ADDRESS} install -o result 2>&1 if ! grep serve result/test.out; then echo 'Expected "serve" result but found "'$(cat result/test.out)'"' exit 1 fi echo echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/000077500000000000000000000000001516554100600257705ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/TARGETS000066400000000000000000000004061516554100600270240ustar00rootroot00000000000000{ "greetlib": {"type": "install", "dirs": [[["TREE", null, "./greetlib"], "."]]} , "pydicts": {"type": "install", "dirs": [[["TREE", null, "./pydicts"], "."]]} , "lib with generated hdr": {"type": "install", "dirs": [[["TREE", null, "./samplelib"], "."]]} } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/000077500000000000000000000000001516554100600275655ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/ROOT000066400000000000000000000000001516554100600302610ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/000077500000000000000000000000001516554100600306735ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/TARGETS000066400000000000000000000002001516554100600317170ustar00rootroot00000000000000{ "greet": { "type": "export" , "target": ["src", "greetlib"] , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/include/000077500000000000000000000000001516554100600323165ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS000066400000000000000000000001661516554100600333550ustar00rootroot00000000000000{ "hdrs": { "type": ["@", "rules", "data", "staged"] , "srcs": [["TREE", null, "."]] , "stage": ["greet"] } } greet.hpp000066400000000000000000000012431516554100600340560ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/include// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include void greet(std::string const& str); just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/src/000077500000000000000000000000001516554100600314625ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS000066400000000000000000000002221516554100600325120ustar00rootroot00000000000000{ "greetlib": { "type": ["@", "rules", "CC", "library"] , "name": ["greet"] , "hdrs": [["include", "hdrs"]] , "srcs": ["greet.cpp"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp000066400000000000000000000013571516554100600333020ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "greet/greet.hpp" #include void greet(std::string const& str) { std::cout << ": " << str << std::endl; } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/main/000077500000000000000000000000001516554100600305115ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/main/TARGETS000066400000000000000000000002331516554100600315430ustar00rootroot00000000000000{ "main": { "type": ["@", "rules", "CC", "binary"] , "name": ["main"] , "srcs": ["main.cpp"] , "private-deps": [["@", "greet", "", "greet"]] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/main/main.cpp000066400000000000000000000013011516554100600321340ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "greet/greet.hpp" int main() { greet("Hello, World!"); return 0; } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/greetlib/repos.json000066400000000000000000000006551516554100600316160ustar00rootroot00000000000000{ "main": "main" , "repositories": { "main": { "repository": {"type": "file", "path": "./main"} , "bindings": {"rules": "rules", "greet": "greet"} } , "greet": { "repository": {"type": "file", "path": "./greet", "pragma": {"to_git": true}} , "bindings": {"rules": "rules"} } , "rules": { "repository": {"type": "file", "path": "", "pragma": {"to_git": true}} } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/000077500000000000000000000000001516554100600274475ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/ROOT000066400000000000000000000000001516554100600301430ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/RULES000066400000000000000000000226551516554100600302760ustar00rootroot00000000000000{ "py_dicts": { "target_fields": ["py_files"] , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} , "expression": { "type": "RESULT" , "artifacts": {"type": "empty_map"} , "runfiles": {"type": "empty_map"} , "provides": { "type": "let*" , "bindings": [ [ "converter" , { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": { "type": "foreach" , "var": "d" , "range": {"type": "FIELD", "name": "converter"} , "body": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "d"} } } } , "runfiles": {"type": "empty_map"} , "provides": {"type": "empty_map"} } } ] , [ "py_files" , { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": { "type": "foreach" , "var": "d" , "range": {"type": "FIELD", "name": "py_files"} , "body": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "d"} } } } , "runfiles": {"type": "empty_map"} , "provides": {"type": "empty_map"} } } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "py2json" , "value": [ { "type": "ABSTRACT_NODE" , "string_fields": { "type": "map_union" , "$1": [ {"type": "singleton_map", "key": "from", "value": ["py"]} , {"type": "singleton_map", "key": "to", "value": ["json"]} ] } , "target_fields": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "converter" , "value": [{"type": "var", "name": "converter"}] } , { "type": "singleton_map" , "key": "dicts" , "value": [{"type": "var", "name": "py_files"}] } ] } , "node_type": "convert" } ] } ] } } } } , "dicts_convert": { "string_fields": ["from", "to"] , "target_fields": ["converter", "dicts"] , "config_vars": ["ext"] , "expression": { "type": "let*" , "bindings": [ ["from", {"type": "join", "$1": {"type": "FIELD", "name": "from"}}] , ["to", {"type": "join", "$1": {"type": "FIELD", "name": "to"}}] , [ "ext" , { "type": "var" , "name": "ext" , "default": { "type": "if" , "cond": {"type": "==", "$1": {"type": "var", "name": "to"}, "$2": "py"} , "then": ".py" , "else": { "type": "if" , "cond": { "type": "==" , "$1": {"type": "var", "name": "to"} , "$2": "json" } , "then": ".json" , "else": ".yaml" } } } ] , [ "dicts" , { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "d" , "range": {"type": "FIELD", "name": "dicts"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "d"}} } } ] , [ "converter" , { "type": "to_subdir" , "subdir": "bin" , "flat": false , "$1": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "converter"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } } ] , [ "converter_bin" , { "type": "join" , "$1": {"type": "keys", "$1": {"type": "var", "name": "converter"}} } ] ] , "body": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "$1": { "type": "foreach_map" , "var_key": "path" , "var_val": "file" , "range": {"type": "var", "name": "dicts"} , "body": { "type": "let*" , "bindings": [ [ "out" , { "type": "change_ending" , "ending": {"type": "var", "name": "ext"} , "$1": {"type": "var", "name": "path"} } ] ] , "body": { "type": "ACTION" , "cmd": [ "/bin/sh" , "-c" , { "type": "join" , "$1": { "type": "++" , "$1": [ [{"type": "var", "name": "converter_bin"}] , [ {"type": "var", "name": "from"} , {"type": "var", "name": "to"} ] , [ {"type": "var", "name": "path"} , ">" , {"type": "var", "name": "out"} ] ] } , "separator": " " } ] , "inputs": { "type": "disjoint_map_union" , "$1": [ {"type": "var", "name": "converter"} , { "type": "singleton_map" , "key": {"type": "var", "name": "path"} , "value": {"type": "var", "name": "file"} } ] } , "outs": [{"type": "var", "name": "out"}] } } } } , "runfiles": {"type": "empty_map"} , "provides": {"type": "empty_map"} } } } , "json_dicts": { "target_fields": ["py_dicts"] , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} , "anonymous": { "from_py": { "provider": "py2json" , "rule_map": {"convert": "dicts_convert"} , "target": "py_dicts" } } , "expression": { "type": "RESULT" , "artifacts": { "type": "disjoint_map_union" , "$1": { "type": "foreach" , "var": "a" , "range": {"type": "FIELD", "name": "from_py"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "a"}} } } , "runfiles": {"type": "empty_map"} , "provides": { "type": "let*" , "bindings": [ [ "converter" , { "type": "VALUE_NODE" , "$1": { "type": "RESULT" , "artifacts": { "type": "map_union" , "$1": { "type": "foreach" , "var": "d" , "range": {"type": "FIELD", "name": "converter"} , "body": { "type": "DEP_ARTIFACTS" , "dep": {"type": "var", "name": "d"} } } } , "runfiles": {"type": "empty_map"} , "provides": {"type": "empty_map"} } } ] , [ "py_nodes" , { "type": "++" , "$1": { "type": "foreach" , "var": "d" , "range": {"type": "FIELD", "name": "py_dicts"} , "body": { "type": "DEP_PROVIDES" , "dep": {"type": "var", "name": "d"} , "provider": "py2json" , "default": {"type": "empty_map"} } } } ] ] , "body": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "json2yaml" , "value": [ { "type": "ABSTRACT_NODE" , "string_fields": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "from" , "value": ["json"] } , {"type": "singleton_map", "key": "to", "value": ["yaml"]} ] } , "target_fields": { "type": "map_union" , "$1": [ { "type": "singleton_map" , "key": "converter" , "value": [{"type": "var", "name": "converter"}] } , { "type": "singleton_map" , "key": "dicts" , "value": {"type": "var", "name": "py_nodes"} } ] } , "node_type": "convert" } ] } ] } } } } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/TARGETS000066400000000000000000000003411516554100600305010ustar00rootroot00000000000000{ "json_from_py": {"type": [".", "json_dicts"], "py_dicts": ["exported_py"]} , "exported_py": {"type": "export", "target": "py_dict_files"} , "py_dict_files": {"type": [".", "py_dicts"], "py_files": ["foo.py", "bar.py"]} } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/bar.py000066400000000000000000000012121516554100600305610ustar00rootroot00000000000000# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # type: ignore {"foo": "", None: ["bar"]} just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/dict_converter.py000077500000000000000000000024701516554100600330410ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import ast import json import yaml # this package has lax type hints from typing import Any if len(sys.argv) < 4: print(f"usage: {sys.argv[0]} [py|json|yaml] [py|json|yaml] ") sys.exit(1) with open(sys.argv[3]) as f: data: Any = {} if sys.argv[1] == "py": data = ast.literal_eval(f.read()) elif sys.argv[1] == "json": data = json.load(f) elif sys.argv[1] == "yaml": data = yaml.load(f, Loader=yaml.Loader) # type: ignore if (sys.argv[2] == "py"): print(data) elif sys.argv[2] == "json": print(json.dumps(data, indent=2)) elif sys.argv[2] == "yaml": print(yaml.dump(data, indent=2)) # type: ignore just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/foo.py000066400000000000000000000012041516554100600306010ustar00rootroot00000000000000# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # type: ignore {"foo": "", 0: 4711} just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/pydicts/repos.json000066400000000000000000000001621516554100600314710ustar00rootroot00000000000000{ "repositories": { "main": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/samplelib/000077500000000000000000000000001516554100600277405ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/samplelib/TARGETS000066400000000000000000000005041516554100600307730ustar00rootroot00000000000000{ "": {"type": "export", "target": "lib"} , "lib": { "type": ["@", "rules", "CC", "library"] , "name": ["foo"] , "srcs": ["foo.cpp"] , "hdrs": ["foo.hpp"] } , "foo.hpp": { "type": "generic" , "outs": ["foo.hpp"] , "cmds": ["echo '// generated file' > foo.hpp", "echo 'int foo(int x);' >> foo.hpp"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/data/samplelib/foo.cpp000066400000000000000000000012421516554100600312260ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "foo.hpp" int foo(int x) { return x*x; } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/export-extern.sh000066400000000000000000000037141516554100600302440ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="$PWD/bin/tool-under-test" readonly JUST_MR="$PWD/bin/mr-tool-under-test" readonly LBRDIR="$TEST_TMPDIR/local-build-root" readonly OUT="$TEST_TMPDIR/out" touch ROOT cat > repos.json <<'EOF' { "repositories": { "": { "repository": {"type": "file", "path": "foo", "pragma": {"to_git": true}} , "bindings": {"bar": "bar"} } , "bar": {"repository": {"type": "file", "path": "bar", "pragma": {"to_git": true}}} } } EOF mkdir bar cat > bar/TARGETS <<'EOF' {"it": {"type": "file_gen", "name": "it", "data": "target it in bar"}} EOF mkdir foo echo -n 'file it in foo' > foo/it cat > foo/TARGETS <<'EOF' { "it": {"type": "file_gen", "name": "it", "data": "target it in foo"} , "local": {"type": "export", "target": ["", "it"]} , "distant": {"type": "export", "target": ["@", "bar", "", "it"]} , "file": {"type": "export", "target": ["FILE", null, "it"]} } EOF CONF=$("${JUST_MR}" --local-build-root "${LBRDIR}" setup '') "${JUST}" install --local-build-root "${LBRDIR}" -C "${CONF}" -o "${OUT}" local 2>&1 cat ${OUT}/it echo grep 'target.*foo' ${OUT}/it echo "${JUST}" install --local-build-root "${LBRDIR}" -C "${CONF}" -o "${OUT}" distant 2>&1 cat ${OUT}/it echo grep 'target.*bar' ${OUT}/it echo "${JUST}" install --local-build-root "${LBRDIR}" -C "${CONF}" -o "${OUT}" file 2>&1 cat ${OUT}/it echo grep 'file.*foo' ${OUT}/it echo just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/serve-sync.sh000066400000000000000000000100511516554100600275060ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly LBR_LOCAL="${TEST_TMPDIR}/lbr.local" readonly LBR="${TEST_TMPDIR}/lbr.with-serve" readonly WRKDIR="${PWD}/work" readonly LOCAL_SRC_DIR="${TEST_TMPDIR}/src" if [ "${COMPATIBLE:-}" = "YES" ]; then ARGS="--compatible" else ARGS="" fi RE_ARGS="${ARGS} -r ${REMOTE_EXECUTION_ADDRESS}" SERVE_ARGS="${ARGS} -R ${SERVE} -r ${REMOTE_EXECUTION_ADDRESS}" mkdir -p "${LOCAL_SRC_DIR}" cd "${LOCAL_SRC_DIR}" git init git config user.name "Nobody" git config user.email "nobody@example.org" cat > TARGETS <<'EOF' { "local (unexported)": { "type": ["@", "rules", "CC", "library"] , "hdrs": ["bar.hpp"] , "deps": [["@", "lib", "", ""]] } , "local": {"type": "export", "target": "local (unexported)"} , "": { "type": ["@", "rules", "CC", "binary"] , "name": ["use"] , "srcs": ["use.cpp"] , "private-deps": ["local", ["@", "lib", "", ""]] } } EOF cat > bar.hpp <<'EOF' int bar(int x) { return x+1;} EOF cat > use.cpp <<'EOF' #include "foo.hpp" #include "bar.hpp" #include int main(int arc, char **argv) { std::cout << "foo(2)=" << foo(2) << "\n"; std::cout << "bar(2)=" << bar(2) << std::endl; return 0; } EOF git add . git commit -m 'Initial commit ... of a repository available locally but not to serve.' readonly COMMIT=$(git log --pretty=%H) mkdir -p "${WRKDIR}" cd "${WRKDIR}" touch ROOT # Set up repositories; everythig is content-fixed # - "lib" is known to serve ahead of time # - "rules" is "to_git", hence will be made known to the serve end point # - "" is only known locally, not to the serve end point cat > repos.json <&1 "${JUST_MR}" --norc --local-build-root "${LBR_LOCAL}" --just "${JUST}" \ -L '["env", "PATH='"${PATH}"'"]' build 2>&1 echo echo 'Serve build' echo # Now, with a completely fresh local build root, build the default target, using # a serve endpoint which can provide ["@", "lib", "", ""], # but not ["@", "", "", "local"]. "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" ${SERVE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' build 2>&1 echo 'remote build (same endpoint)' echo # Now, when continuing without serve (but still using remote-exection), things # should still be in a consistent state, without causing staging conflicts. echo "${JUST_MR}" --norc --local-build-root "${LBR}" --just "${JUST}" ${RE_ARGS} \ -L '["env", "PATH='"${PATH}"'"]' build 2>&1 echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-cache/target-cache-hit.sh000066400000000000000000000050701516554100600305260ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu readonly JUST="$PWD/bin/tool-under-test" readonly JUST_MR="$PWD/bin/mr-tool-under-test" readonly LBRDIR="$TEST_TMPDIR/local-build-root" readonly TESTDIR="$TEST_TMPDIR/test-root" # create project files including an exported target mkdir -p "$TESTDIR" cd "$TESTDIR" touch ROOT cat > repos.json <ppp readonly LOW=100000000 readonly HIGH=999999999 readonly RND="p$(hostname)p$(date +%s%N)p$$p$(shuf -i $LOW-$HIGH -n 1)" cat > TARGETS <&1 "$JUST" build -L '["env", "PATH='"${PATH}"'"]' -C "$CONF" main --local-build-root="$LBRDIR" $ARGS 2>&1 REMOTE_EXECUTION_ARGS="" if [ "${REMOTE_EXECUTION_ADDRESS:-}" != "" ]; then REMOTE_EXECUTION_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then REMOTE_EXECUTION_PROPS="$(printf " --remote-execution-property %s" ${REMOTE_EXECUTION_PROPERTIES})" REMOTE_EXECUTION_ARGS="${REMOTE_EXECUTION_ARGS} ${REMOTE_EXECUTION_PROPS}" fi fi # build project twice remotely to trigger a target cache hit "$JUST" build -L '["env", "PATH='"${PATH}"'"]' -C "$CONF" main --local-build-root="$LBRDIR" $ARGS $REMOTE_EXECUTION_ARGS 2>&1 "$JUST" build -L '["env", "PATH='"${PATH}"'"]' -C "$CONF" main --local-build-root="$LBRDIR" $ARGS $REMOTE_EXECUTION_ARGS 2>&1 just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/000077500000000000000000000000001516554100600251565ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/TARGETS000066400000000000000000000030041516554100600262070ustar00rootroot00000000000000{ "upwards reference": { "type": ["@", "rules", "shell/test", "script"] , "name": ["upwards"] , "test": ["upwards.sh"] , "deps": [["", "tool-under-test"]] } , "repository naming": { "type": ["@", "rules", "shell/test", "script"] , "name": ["repo_names"] , "test": ["repo_names.sh"] , "deps": [["", "tool-under-test"]] } , "resolution of built-in rules": { "type": ["@", "rules", "shell/test", "script"] , "name": ["built-in-resolution"] , "test": ["built-in-resolution.sh"] , "deps": [["", "tool-under-test"]] } , "glob expansion": { "type": ["@", "rules", "shell/test", "script"] , "name": ["glob"] , "test": ["glob.sh"] , "deps": [["", "tool-under-test"]] } , "configure target name": { "type": ["@", "rules", "shell/test", "script"] , "name": ["configure-target"] , "test": ["configure-target.sh"] , "deps": [["", "tool-under-test"]] } , "configure variables": { "type": ["@", "rules", "shell/test", "script"] , "name": ["configure-vars"] , "test": ["configure-vars.sh"] , "deps": [["", "tool-under-test"]] } , "tree inputs": { "type": ["@", "rules", "shell/test", "script"] , "name": ["tree-inputs"] , "test": ["tree-inputs.sh"] , "deps": [["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["target-tests"] , "deps": [ "configure target name" , "configure variables" , "glob expansion" , "repository naming" , "resolution of built-in rules" , "tree inputs" , "upwards reference" ] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/built-in-resolution.sh000077500000000000000000000025251516554100600314450ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR=$(realpath out) mkdir -p src/some/module touch src/ROOT cd src/some/module cat > RULES <<'EOI' { "install": { "expression": { "type": "RESULT" , "artifacts": { "type": "singleton_map" , "key": "some internal path" , "value": {"type": "BLOB", "data": "FROM USER-DEFINED INSTALL"} } } } } EOI cat > TARGETS <<'EOI' { "user": {"type": ["./", ".", "install"]} , "built-in": {"type": "install", "files": {"the/public/path": "user"}} } EOI echo ${TOOL} install built-in --local-build-root ${BUILDROOT} -o ${OUTDIR} 2>&1 echo grep 'FROM USER-DEFINED INSTALL' ${OUTDIR}/the/public/path echo echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/configure-target.sh000066400000000000000000000047261516554100600307700ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR=$(realpath out) mkdir src cd src touch ROOT cat > TARGETS <<'EOI' { "configured": { "type": "configure" , "arguments_config": ["TARGET", "FOO", "BAR"] , "target": {"type": "var", "name": "TARGET"} , "config": { "type": "let*" , "bindings": [ ["FOO", {"type": "var", "name": "FOO", "default": "foo"}] , ["BAR", {"type": "var", "name": "BAR", "default": "bar"}] , [ "FOOBAR" , { "type": "join" , "$1": [{"type": "var", "name": "FOO"}, {"type": "var", "name": "BAR"}] } ] ] , "body": {"type": "env", "vars": ["FOO", "BAR", "FOOBAR"]} } } , "foobar": { "type": "generic" , "outs": ["out.txt"] , "arguments_config": ["FOOBAR"] , "cmds": [ { "type": "join" , "separator": " " , "$1": [ "echo" , "-n" , {"type": "join_cmd", "$1": [{"type": "var", "name": "FOOBAR"}]} , "> out.txt" ] } ] } , "bar": { "type": "file_gen" , "arguments_config": ["BAR"] , "name": "out.txt" , "data": {"type": "var", "name": "BAR"} } } EOI echo -n unrelated > out.txt echo ${TOOL} install --local-build-root "${BUILDROOT}" -o "${OUTDIR}" \ -D '{"TARGET": "foobar", "FOO": "_F_O_O_"}' configured 2>&1 echo cat "${OUTDIR}/out.txt" echo [ $(cat "${OUTDIR}/out.txt") = "_F_O_O_bar" ] echo ${TOOL} install --local-build-root "${BUILDROOT}" -o "${OUTDIR}" \ -D '{"TARGET": "bar", "FOO": "_F_O_O_"}' configured 2>&1 echo cat "${OUTDIR}/out.txt" echo [ $(cat "${OUTDIR}/out.txt") = "bar" ] echo ${TOOL} install --local-build-root "${BUILDROOT}" -o "${OUTDIR}" \ -D '{"TARGET": "out.txt", "FOO": "_F_O_O_"}' configured 2>&1 echo cat "${OUTDIR}/out.txt" echo [ $(cat "${OUTDIR}/out.txt") = "unrelated" ] echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/configure-vars.sh000066400000000000000000000036321516554100600304500ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR=$(realpath out) mkdir src cd src touch ROOT cat > TARGETS <<'EOI' { "copy foo": { "type": "configure" , "arguments_config": ["FOO"] , "target": "use bar" , "config": { "type": "let*" , "bindings": [["BAR", {"type": "var", "name": "FOO"}], ["FOO", "REDACTED"]] , "body": {"type": "env", "vars": ["BAR", "FOO"]} } } , "use bar": { "type": "file_gen" , "arguments_config": ["BAR"] , "data": {"type": "var", "name": "BAR", "default": "bar"} , "name": "out.txt" } } EOI echo ${TOOL} analyse --local-build-root "${BUILDROOT}" \ --dump-vars "${OUTDIR}/vars.json" \ --dump-targets "${OUTDIR}/targets.json" \ -D '{"FOO": "the value", "BAR": "unused!"}' 'copy foo' 2>&1 echo cat "${OUTDIR}/vars.json" echo cat "${OUTDIR}/targets.json" echo copy_configs=$(cat "${OUTDIR}/targets.json" | jq -acM '."@" | ."" | ."" | ."copy foo"') echo "${copy_configs}" [ $(echo "${copy_configs}" | jq length) -eq 1 ] config=$(echo "${copy_configs}" | jq '.[0]') echo $config [ $(echo "$config" | jq -acM 'keys') = '["FOO"]' ] echo "$config" | jq -acM '."FOO"' [ "$(echo "$config" | jq -acM '."FOO"')" = '"the value"' ] [ "$(cat "${OUTDIR}/vars.json")" = '["FOO"]' ] echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/glob.sh000077500000000000000000000126041516554100600264430ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e TOOL=$(realpath ./bin/tool-under-test) mkdir -p .root BUILDROOT=$(realpath .root) mkdir -p out OUTDIR_BASE=$(realpath out) echo {} > repos.json export CONF=$(realpath repos.json) mkdir src cd src touch ROOT echo -n A > a.txt chmod 644 a.txt echo -n B > b.txt echo -n C > c.txt chmod 755 b.txt mkdir d.txt mkdir foo echo too deep > foo/a.txt cat > RULES <<'EOI' { "reflect": { "target_fields": ["deps"] , "expression": { "type": "let*" , "bindings": [ [ "inputs" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "deps"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } ] , [ "keys" , { "type": "singleton_map" , "key": "keys.txt" , "value": { "type": "BLOB" , "data": { "type": "join" , "separator": " " , "$1": {"type": "keys", "$1": {"type": "var", "name": "inputs"}} } } } ] , [ "content" , { "type": "ACTION" , "inputs": { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "inputs"} } , "outs": ["content.txt"] , "cmd": [ "sh" , "-c" , { "type": "join" , "separator": " " , "$1": [ "mkdir -p work && cd work &&" , { "type": "join_cmd" , "$1": { "type": "++" , "$1": [ ["cat"] , { "type": "keys" , "$1": {"type": "var", "name": "inputs"} } ] } } , "> ../content.txt" ] } ] } ] , [ "outputs" , { "type": "map_union" , "$1": [ {"type": "var", "name": "keys"} , {"type": "var", "name": "content"} ] } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "outputs"}} } } } EOI cat > TARGETS <<'EOI' { "b.txt": {"type": "file_gen", "name": "b-new.txt", "data": "fromtarget"} , "enumeration": {"type": "reflect", "deps": ["a.txt", "b.txt", "c.txt"]} , "glob": {"type": "reflect", "deps": [["GLOB", null, "*.txt"]]} , "with_target": {"type": "reflect", "deps": [["GLOB", null, "*.txt"], "b.txt"]} , "not_top_level": {"type": "reflect", "deps": [["GLOB", null, "foo/*.txt"]]} } EOI do_test() { echo === Enumeration refers to targets === ${TOOL} install -L '["env", "PATH='"${PATH}"'"]' -C ${CONF} --local-build-root ${BUILDROOT} -o ${OUTDIR}/enum enumeration 2>&1 cat ${OUTDIR}/enum/keys.txt echo cat ${OUTDIR}/enum/content.txt echo [ "$(cat ${OUTDIR}/enum/keys.txt)" = "a.txt b-new.txt c.txt" ] [ "$(cat ${OUTDIR}/enum/content.txt)" = "AfromtargetC" ] echo === Glob always refers to files and directories are ignored === ${TOOL} install -L '["env", "PATH='"${PATH}"'"]' -C ${CONF} --local-build-root ${BUILDROOT} -o ${OUTDIR}/glob glob 2>&1 cat ${OUTDIR}/glob/keys.txt echo cat ${OUTDIR}/glob/content.txt echo [ "$(cat ${OUTDIR}/glob/keys.txt)" = "a.txt b.txt c.txt" ] [ "$(cat ${OUTDIR}/glob/content.txt)" = "ABC" ] echo === Globs and targets can be combined === ${TOOL} install -L '["env", "PATH='"${PATH}"'"]' -C ${CONF} --local-build-root ${BUILDROOT} -o ${OUTDIR}/with_target with_target 2>&1 cat ${OUTDIR}/with_target/keys.txt echo cat ${OUTDIR}/with_target/content.txt echo [ "$(cat ${OUTDIR}/with_target/keys.txt)" = "a.txt b-new.txt b.txt c.txt" ] [ "$(cat ${OUTDIR}/with_target/content.txt)" = "AfromtargetBC" ] echo === Globs only inspect the top-level directory of the module === ${TOOL} install -L '["env", "PATH='"${PATH}"'"]' -C ${CONF} --local-build-root ${BUILDROOT} -o ${OUTDIR}/not_top_level not_top_level 2>&1 cat ${OUTDIR}/not_top_level/keys.txt echo cat ${OUTDIR}/not_top_level/content.txt echo [ -z "$(cat ${OUTDIR}/not_top_level/keys.txt)" ] [ -z "$(cat ${OUTDIR}/not_top_level/content.txt)" ] echo === Done === } echo echo '***** Tests on the plain file system *****' echo OUTDIR=${OUTDIR_BASE}/file_system do_test echo echo '***** Tests on a git root *****' echo git init git add . git config user.name "N.O.Body" git config user.email "nobody@example.com" git commit -m 'Initial commit' REPO=$(realpath .git) TREE=$(git log -n1 --format='%T') cat > $CONF < repo_A/data.txt cat > repo_A/TARGETS <<'EOF' { "hello": { "type": "generic" , "deps": [ "data.txt" ] , "outs": [ "hello.txt" ] , "cmds": [ "echo Hello `cat data.txt` > hello.txt" ] } , "use": { "type": "generic" , "deps": [ [ "@" , "other" , "." , "hello" ] ] , "outs": [ "use.txt" ] , "cmds": [ "cat hello.txt > use.txt" , "echo This is A >> use.txt" ] } , "back": {"type": "generic" , "deps": [ ["@", "other", ".", "use"] ] , "outs": [ "back.txt"] , "cmds" : [ "echo I am A and I see the following file > back.txt" , "echo >> back.txt" , "cat use.txt >> back.txt" ] } } EOF mkdir repo_B echo 'B' > repo_B/data.txt cat > repo_B/TARGETS <<'EOF' { "hello": { "type": "generic" , "deps": [ "data.txt" ] , "outs": [ "hello.txt" ] , "cmds": [ "echo Hello `cat data.txt` > hello.txt" ] } , "use": { "type": "generic" , "deps": [ [ "@" , "other" , "." , "hello" ] ] , "outs": [ "use.txt" ] , "cmds": [ "cat hello.txt > use.txt" , "echo This is B >> use.txt" ] } , "back": {"type": "generic" , "deps": [ ["@", "other", ".", "use"] ] , "outs": [ "back.txt"] , "cmds" : [ "echo I am B and I see the following file > back.txt" , "echo >> back.txt" , "cat use.txt >> back.txt" ] } } EOF cat > bindings.json <<'EOF' { "repositories" : {"A" : { "workspace_root": ["file", "repo_A"] , "bindings": {"other": "B"} } , "B": { "workspace_root": ["file", "repo_B"] , "bindings": {"other": "A"} } } } EOF mkdir -p .root echo == Building in A == ./bin/tool-under-test install -L '["env", "PATH='"${PATH}"'"]' -C bindings.json -o . --local-build-root .root --main A . back 2>&1 cat back.txt grep -q 'I am A' back.txt grep -q 'This is B' back.txt grep -q 'Hello A' back.txt rm -f back.txt echo == Building in B == ./bin/tool-under-test install -L '["env", "PATH='"${PATH}"'"]' -C bindings.json -o . --local-build-root .root --main B . back 2>&1 cat back.txt grep -q 'I am B' back.txt grep -q 'This is A' back.txt grep -q 'Hello B' back.txt just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/tree-inputs.sh000077500000000000000000000022611516554100600277750ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir -p .root mkdir -p src touch src/ROOT # Check if we can collect and stage empty trees cat > src/TARGETS <<'EOF' { "make_trees": { "type": "generic" , "cmds": ["mkdir -p foo bar/baz"] , "out_dirs": ["."] } , "read_trees": { "type": "generic" , "deps": ["make_trees"] , "cmds": ["set -e", "ls -l foo", "ls -l bar/baz", "echo SUCCESS > result"] , "outs": ["result"] } } EOF ./bin/tool-under-test install -o out --workspace-root src \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root .root . read_trees 2>&1 grep SUCCESS out/result echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/target-tests/upwards.sh000077500000000000000000000043361516554100600272100ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir -p .root mkdir -p src touch src/ROOT echo BAD > outside.txt # Target names that look like upwards references are OK echo "== legitimate target ==" cat > src/TARGETS <<'EOF' { "it": { "type": "generic" , "cmds": ["cat *.txt > out"] , "outs": ["out"] , "deps": ["../outside.txt"] } , "../outside.txt": {"type": "file_gen", "name": "inside.txt", "data": "Everything OK"} } EOF ./bin/tool-under-test install -o out --workspace-root src \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root .root . it 2>&1 grep OK out/out grep BAD out/out && exit 1 || : # Upwards references in relative target names are OK; # continue the previous test case echo "== legitimate upwards target reference ==" mkdir -p src/deep cat > src/deep/TARGETS <<'EOF' { "OK up": { "type": "generic" , "cmds": ["cat *.txt > out"] , "outs": ["out"] , "deps": [ ["./", "..", "../outside.txt"] ] } } EOF ./bin/tool-under-test install -o out2 --workspace-root src \ -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root .root deep 'OK up' 2>&1 grep OK out2/out # Upwards refernces and targets outside the repo are not OK cat > TARGETS <<'EOF' { "outside.txt": {"type": "file_gen", "name": "inside.txt", "data": "BAD"}} EOF for REF in '"../outside.txt"' \ '["FILE", ".", "../outside.txt"]' \ '["./", "..", "outside.txt"]' do echo "== true upwards reference $REF ==" cat > src/TARGETS < out"] , "outs": ["out"] , "deps": [${REF}] } } EOF cat src/TARGETS ./bin/tool-under-test analyse --workspace-root src . it 2>&1 && exit 1 || : done echo DONE just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/000077500000000000000000000000001516554100600255255ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/TARGETS000066400000000000000000000033601516554100600265630ustar00rootroot00000000000000{ "basic": { "type": ["@", "rules", "shell/test", "script"] , "name": ["basic"] , "test": ["basic.sh"] , "deps": [["", "tool-under-test"]] } , "mr_setup": { "type": ["@", "rules", "shell/test", "script"] , "name": ["mr_setup"] , "test": ["mr_setup.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "precomputed_indirection": { "type": ["@", "rules", "shell/test", "script"] , "name": ["precomputed_indirection"] , "test": ["precomputed_indirection.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] } , "absent_tree_structure_foo (repo)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": [ "NESTED_DIR='src/nested_dir/nested_dir_2'" , "mkdir -p ${NESTED_DIR}" , "echo foo > ${NESTED_DIR}/file.txt" ] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "absent_tree_structure_bar (repo)": { "type": "generic" , "arguments_config": ["TEST_ENV"] , "out_dirs": ["src"] , "cmds": [ "NESTED_DIR='src/nested_dir/nested_dir_2'" , "mkdir -p ${NESTED_DIR}" , "echo bar > src/nested_dir/nested_dir_2/file.txt" ] , "env": {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} } , "absent_tree_structure": { "type": ["end-to-end", "with serve"] , "name": ["absent_tree_structure"] , "test": ["absent_tree_structure.sh"] , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] , "repos": ["absent_tree_structure_foo (repo)", "absent_tree_structure_bar (repo)"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["tree-structure"] , "deps": ["absent_tree_structure", "basic", "mr_setup", "precomputed_indirection"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/absent_tree_structure.sh000066400000000000000000000140441516554100600324770ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly RCFILE="${TEST_TMPDIR}/mrrc.json" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi readonly LOCAL_REPO="${ROOT}/local" mkdir -p "${LOCAL_REPO}/src/foo/nested_foo" echo foo > "${LOCAL_REPO}/src/foo/nested_foo/file.txt" mkdir -p "${LOCAL_REPO}/src/bar/nested_bar" echo bar > "${LOCAL_REPO}/src/bar/nested_bar/file.txt" readonly MAIN_REPO="${ROOT}/main" mkdir -p "${MAIN_REPO}" cd "${MAIN_REPO}" touch ROOT cat > "TARGETS" << 'EOF' { "": {"type": "export", "target": "generate"} , "generate": { "type": "generic" , "deps": [["TREE", null, "src"]] , "outs": ["result.txt"] , "cmds": ["ls -R src > result.txt"] } } EOF echo echo TARGETS: cat TARGETS cat > repo-config.json < "${RCFILE}" <<'EOF' {"absent": [{"root": "workspace", "path": "absent.json"}]} EOF echo echo mrrc.json: cat "${RCFILE}" # Test absent tree structure root of an absent root: # Set foo and structure foo absent: cat > absent.json <<'EOF' ["foo", "structure_foo"] EOF echo echo "Absent tree structure root of an absent root. Expected to be computed on serve:" ("${JUST_MR}" --rc "${RCFILE}" \ --local-build-root "${LBRDIR}/absent_absent" -C repo-config.json \ -r "${REMOTE_EXECUTION_ADDRESS}" -R "${SERVE}" ${COMPAT} \ --main result_foo -L '["env", "PATH='"${PATH}"'"]' --log-limit 4 \ --just "${JUST}" install -o "${OUT}/absent_absent" 2>&1) \ > "${OUT}/log_absent_absent" echo cat "${OUT}/log_absent_absent" grep 'Root \["tree structure", "foo", {"absent": true}\] was computed on serve' \ "${OUT}/log_absent_absent" echo cat "${OUT}/absent_absent/result.txt" # Test local tree structure of absent roots: # Set both source repositories foo and bar absent: cat > absent.json <<'EOF' ["foo", "bar"] EOF echo echo "Local tree structure root of an absent root." echo "Expected to be computed on serve and downloaded:" ("${JUST_MR}" --rc "${RCFILE}" \ --local-build-root "${LBRDIR}/local" -C repo-config.json \ -r "${REMOTE_EXECUTION_ADDRESS}" -R "${SERVE}" ${COMPAT} \ --main result_foo -L '["env", "PATH='"${PATH}"'"]' --log-limit 4 \ --just "${JUST}" install -o "${OUT}/result_foo" 2>&1) \ > "${OUT}/log_result_foo" echo cat "${OUT}/log_result_foo" grep 'Root \["tree structure", "foo"\] has been downloaded from the remote end point' \ "${OUT}/log_result_foo" echo cat "${OUT}/result_foo/result.txt" echo echo "Local tree structure root of an absent root." echo "Expected to be taken from local cache:" ("${JUST_MR}" --rc "${RCFILE}" \ --local-build-root "${LBRDIR}/local" -C repo-config.json \ -r "${REMOTE_EXECUTION_ADDRESS}" -R "${SERVE}" ${COMPAT} \ --main result_bar -L '["env", "PATH='"${PATH}"'"]' --log-limit 4 \ --just "${JUST}" install -o "${OUT}/result_bar" 2>&1) \ > "${OUT}/log_result_bar" echo cat "${OUT}/log_result_bar" # The repo "bar" is a structural copy of "foo" with a different content of # the file, so tree_structure(foo) == tree_structure(bar). # Given the fact that tree_structure(foo) has been already evaluated # and downloaded from the remote end point, there is no need to download it # one more time. tree_structure(bar) gets taken from the local cache. grep 'Root \["tree structure", "bar"\] has been taken from local cache' \ "${OUT}/log_result_bar" echo cat "${OUT}/result_bar/result.txt" # Test absent tree structure root of a local root. # Set only tree structure absent: cat > absent.json <<'EOF' ["structure_local"] EOF echo echo "Absent tree structure root of a local root." echo "Expected to be computed locally and uploaded to serve:" ("${JUST_MR}" --rc "${RCFILE}" \ --local-build-root "${LBRDIR}/absent_local" -C repo-config.json \ -r "${REMOTE_EXECUTION_ADDRESS}" -R "${SERVE}" ${COMPAT} \ --main result_local -L '["env", "PATH='"${PATH}"'"]' --log-limit 4 \ --just "${JUST}" install -o "${OUT}/absent_local" 2>&1) \ > "${OUT}/log_absent_local" echo cat "${OUT}/log_absent_local" grep 'Root \["tree structure", "local", {"absent": true}\] was computed locally and uploaded to serve' \ "${OUT}/log_absent_local" echo cat "${OUT}/absent_local/result.txt" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/basic.sh000066400000000000000000000116551516554100600271520ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi readonly TARGET_ROOT="${ROOT}/target_root" mkdir -p "${TARGET_ROOT}" cd "${TARGET_ROOT}" cat > "TARGETS.install_structure" << 'EOF' { "": {"type": "export", "target": "test"} , "test": { "type": "install" , "deps": [["TREE", null, "test_dir"]] } } EOF echo "TARGETS.install_structure": cat "TARGETS.install_structure" echo cat > "TARGETS.ls_structure" << 'EOF' { "": {"type": "export", "target": "generate"} , "generate": { "type": "generic" , "deps": [["TREE", null, "test_dir"]] , "outs": ["result.txt"] , "cmds": ["ls -R test_dir > result.txt"] } } EOF echo "TARGETS.ls_structure": cat "TARGETS.ls_structure" echo echo "Backing up the target_root to git:" git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 git add . 2>&1 git commit -m "TR: initial commit" 2>&1 readonly TR_GIT_TREE=$(git log -n 1 --format="%T") # Create source directory and init git: readonly SRC_DIR="${ROOT}/src" mkdir -p "${SRC_DIR}" cd "${SRC_DIR}" git init 2>&1 git branch -m stable-1.0 2>&1 git config user.email "nobody@example.org" 2>&1 git config user.name "Nobody" 2>&1 readonly NESTED_DIR_2="${SRC_DIR}/test_dir/nested_dir/nested_dir_2" readonly FILE_TO_BE_CHANGED="${NESTED_DIR_2}/file" mkdir -p "${NESTED_DIR_2}" echo "initial content" > "${FILE_TO_BE_CHANGED}" # Obtain the initial git tree: git add . 2>&1 git commit -m "Initial commit" 2>&1 readonly INITIAL_GIT_TREE=$(git log -n 1 --format="%T") echo "tree hash: ${INITIAL_GIT_TREE}" # Obtain the updated git tree: echo "updated content" > "${FILE_TO_BE_CHANGED}" git add . 2>&1 git commit -m "Update file" 2>&1 readonly UPDATED_GIT_TREE=$(git log -n 1 --format="%T") echo "tree hash: ${UPDATED_GIT_TREE}" readonly MAIN_ROOT="${ROOT}/main" mkdir -p "${MAIN_ROOT}" cd "${MAIN_ROOT}" echo "Creating repo-config at ${MAIN_ROOT}:" cat > repo-config.json << EOF { "repositories": { "initial_sources": { "workspace_root": ["git tree", "${INITIAL_GIT_TREE}", "${SRC_DIR}/.git"] , "target_root": ["git tree", "${TR_GIT_TREE}", "${TARGET_ROOT}/.git"] , "target_file_name": "TARGETS.install_structure" } , "tree_structure_1": { "workspace_root": ["tree structure", "initial_sources"] , "target_root": ["git tree", "${TR_GIT_TREE}", "${TARGET_ROOT}/.git"] , "target_file_name": "TARGETS.ls_structure" } , "updated_sources": { "workspace_root": ["git tree", "${UPDATED_GIT_TREE}", "${SRC_DIR}/.git"] , "target_root": ["git tree", "${TR_GIT_TREE}", "${TARGET_ROOT}/.git"] , "target_file_name": "TARGETS.install_structure" } , "tree_structure_2": { "workspace_root": ["tree structure", "updated_sources"] , "target_root": ["git tree", "${TR_GIT_TREE}", "${TARGET_ROOT}/.git"] , "target_file_name": "TARGETS.ls_structure" } } } EOF cat repo-config.json echo # initial_sources and updated_sources have obviously different workspace roots: # the files have different content, so the respective trees are different # as well. # BUT tree_structure_1 and tree_structure_2 produce the same result, since # they rely on the "tree structure" of the sources, which are the same. # Even so tree_structure_2 doesn't get built directly, it still can be taken # from cache, since tree_structure_1 is keyed in the same way as # tree_structure_2 is. echo "Building tree_structure_1 (expected a new cache entry):" ("${JUST}" install -L '["env", "PATH='"${PATH}"'"]' "${COMPAT}" \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main tree_structure_1 --log-limit 4 -o "${OUT}/tree_structure_1" 2>&1) > \ "${OUT}/log" echo cat "${OUT}/log" echo grep 'Export target \["@","tree_structure_1","",""\] registered for caching' \ "${OUT}/log" echo "Building tree_structure_2 (expected to be taken from cache):" ("${JUST}" install -L '["env", "PATH='"${PATH}"'"]' "${COMPAT}" \ --local-build-root "${LBRDIR}" -C repo-config.json \ --main tree_structure_2 --log-limit 4 -o "${OUT}/tree_structure_2" 2>&1) > \ "${OUT}/log2" echo cat "${OUT}/log2" echo grep 'Export target \["@","tree_structure_2","",""\] taken from cache' \ "${OUT}/log2" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/mr_setup.sh000066400000000000000000000121501516554100600277160ustar00rootroot00000000000000#!/bin/sh # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi readonly TARGET_ROOT="${ROOT}/target_root" mkdir -p "${TARGET_ROOT}" cd "${TARGET_ROOT}" cat > "TARGETS.install_structure" << 'EOF' { "": {"type": "export", "target": "test"} , "test": { "type": "install" , "deps": [["TREE", null, "test_dir"]] } } EOF echo "TARGETS.install_structure": cat "TARGETS.install_structure" echo cat > "TARGETS.ls_structure" << 'EOF' { "": {"type": "export", "target": "generate"} , "generate": { "type": "generic" , "deps": [["TREE", null, "test_dir"]] , "outs": ["result.txt"] , "cmds": ["ls -R test_dir > result.txt"] } } EOF echo "TARGETS.ls_structure": cat "TARGETS.ls_structure" echo # Create source directories that have the same structure, but different content # of the file: readonly SRC_DIR="${ROOT}/src" readonly NESTED_DIR="test_dir/nested_dir/nested_dir_2" mkdir -p "${SRC_DIR}/${NESTED_DIR}" echo "initial content" > "${SRC_DIR}/${NESTED_DIR}/file" readonly SRC_DIR_2="${ROOT}/src2" mkdir -p "${SRC_DIR}/${NESTED_DIR}" cp -R "${SRC_DIR}" "${SRC_DIR_2}" echo "updated content" > "${SRC_DIR_2}/${NESTED_DIR}/file" readonly MAIN_ROOT="${ROOT}/main" mkdir -p "${MAIN_ROOT}" cd "${MAIN_ROOT}" echo "Creating repo-config at ${MAIN_ROOT}:" cat > repo-config.json << EOF { "repositories": { "targets": { "repository": { "type": "file" , "path": "${TARGET_ROOT}" , "pragma": {"to_git": true} } } , "initial_sources": { "repository": { "type": "file" , "path": "${SRC_DIR}" , "pragma": {"to_git": true} } , "target_root": "targets" , "target_file_name": "TARGETS.install_structure" } , "updated_sources": { "repository": { "type": "file" , "path": "${SRC_DIR_2}" , "pragma": {"to_git": true} } , "target_root": "targets" , "target_file_name": "TARGETS.install_structure" } , "tree_structure_1": { "repository": { "type": "tree structure" , "repo": "initial_sources" } , "target_root": "targets" , "target_file_name": "TARGETS.ls_structure" } , "tree_structure_2": { "repository": { "type": "tree structure" , "repo": "updated_sources" } , "target_root": "targets" , "target_file_name": "TARGETS.ls_structure" } } } EOF cat repo-config.json echo # initial_sources and updated_sources have obviously different workspace roots: # the files have different content, so the respective trees are different # as well. # BUT tree_structure_1 and tree_structure_2 produce the same result, since # they rely on the "tree structure" of the sources, which are the same. # Even so tree_structure_2 doesn't get built directly, it still can be taken # from cache, since tree_structure_1 is keyed in the same way as # tree_structure_2 is. echo "JustMR setup for tree_structure_1:" readonly CONF_1=$("${JUST_MR}" --norc --local-build-root "${LBRDIR}" \ -C repo-config.json --main tree_structure_1 setup) cat "${CONF_1}" echo echo "Building tree_structure_1 (expected a new cache entry):" echo ("${JUST_MR}" --norc --local-build-root "${LBRDIR}" -C repo-config.json \ --main tree_structure_1 --just "${JUST}" \ install -L '["env", "PATH='"${PATH}"'"]' "${COMPAT}" \ --log-limit 4 -o "${OUT}/tree_structure_1" 2>&1) > "${OUT}/log" cat "${OUT}/log" echo grep 'Export target \["@","tree_structure_1","",""\] registered for caching' \ "${OUT}/log" echo "JustMR setup for tree_structure_2:" readonly CONF_2=$("${JUST_MR}" --norc --local-build-root "${LBRDIR}" \ -C repo-config.json --main tree_structure_2 setup) cat "${CONF_2}" echo echo "Building tree_structure_2 (expected to be taken from cache):" echo ("${JUST_MR}" --norc --local-build-root "${LBRDIR}" -C repo-config.json \ --main tree_structure_2 --just "${JUST}" \ install -L '["env", "PATH='"${PATH}"'"]' "${COMPAT}" \ --log-limit 4 -o "${OUT}/tree_structure_2" 2>&1) > "${OUT}/log2" cat "${OUT}/log2" echo grep 'Export target \["@","tree_structure_2","",""\] taken from cache' \ "${OUT}/log2" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/tree-structure/precomputed_indirection.sh000066400000000000000000000053111516554100600327770ustar00rootroot00000000000000#!/bin/sh # Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="$(pwd)" readonly LBRDIR="$TMPDIR/local-build-root" readonly JUST="${ROOT}/bin/tool-under-test" readonly JUST_MR="${ROOT}/bin/mr-tool-under-test" readonly OUT="${ROOT}/out" mkdir -p "${OUT}" COMPAT="" if [ "${COMPATIBLE:-}" = "YES" ]; then COMPAT="--compatible" fi readonly SRC_DIR="${ROOT}/src" mkdir -p "${SRC_DIR}/test_dir/nested_dir/nested_dir_2" echo "initial content" > "${SRC_DIR}/test_dir/nested_dir/nested_dir_2/file" readonly MAIN_ROOT="${ROOT}/main" mkdir -p "${MAIN_ROOT}" cd "${MAIN_ROOT}" cat > TARGETS << 'EOF' { "" : {"type": "export", "target": "ls"} , "ls": { "type": "generic" , "deps": [["TREE", null, "test_dir"]] , "outs": ["result.txt"] , "cmds": ["ls -R test_dir > result.txt"] } } EOF echo "Creating repo-config at ${MAIN_ROOT}:" cat > repo-config.json << EOF { "repositories": { "src": { "repository": { "type": "file" , "path": "${SRC_DIR}" , "pragma": {"to_git": true} } } , "src_indirection": { "repository": "src" } , "src_structure": { "repository": { "type": "tree structure" , "repo": "src_indirection" } } , "src_structure_indirection": { "repository": "src_structure" } , "src_structure_2": { "repository": { "type": "tree structure" , "repo": "src_structure_indirection" } } , "targets": { "repository": { "type": "file" , "path": "." , "pragma": {"to_git": true} } } , "src_structure_indirection_2": { "repository": "src_structure_2" } , "src_structure_indirection_3": { "repository": "src_structure_indirection_2" } , "result": { "repository": "src_structure_indirection_3" , "target_root": "targets" } } } EOF cat repo-config.json echo echo "JustMR setup:" readonly CONF=$("${JUST_MR}" --norc --local-build-root "${LBRDIR}" \ -C repo-config.json --main result setup) cat "${CONF}" echo echo "Build:" echo "${JUST}" install "${COMPAT}" -L '["env", "PATH='"${PATH}"'"]' \ --local-build-root "${LBRDIR}" -C "${CONF}" -o "${OUT}/result" 2>&1 echo cat "${OUT}/result/result.txt" echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/user-errors/000077500000000000000000000000001516554100600250205ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/end-to-end/user-errors/TARGETS000066400000000000000000000007231516554100600260560ustar00rootroot00000000000000{ "flat-stage": { "type": ["@", "rules", "shell/test", "script"] , "name": ["flat-stage"] , "test": ["flat-stage.sh"] , "deps": [["", "tool-under-test"]] } , "json-errors": { "type": ["@", "rules", "shell/test", "script"] , "name": ["json-errors"] , "test": ["json-errors.sh"] , "deps": [["", "tool-under-test"]] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["user-errors"] , "deps": ["flat-stage", "json-errors"] } } just-buildsystem-justbuild-b1fb5fa/test/end-to-end/user-errors/flat-stage.sh000077500000000000000000000042751516554100600274160ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e mkdir .tool-root touch ROOT mkdir subdir touch foo.txt subdir/foo.txt cat > RULES <<'EOI' { "data": { "target_fields": ["srcs"] , "string_fields": ["flat"] , "expression": { "type": "let*" , "bindings": [ [ "srcs" , { "type": "map_union" , "$1": { "type": "foreach" , "var": "x" , "range": {"type": "FIELD", "name": "srcs"} , "body": {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} } } ] , ["internal information", "DeBuG-InFoRmAtIoN"] , [ "result" , { "type": "to_subdir" , "subdir": "data" , "flat": {"type": "FIELD", "name": "flat"} , "msg": [ "DataRuleSpecificErrorMessage" , {"type": "var", "name": "internal information"} ] , "$1": {"type": "var", "name": "srcs"} } ] ] , "body": {"type": "RESULT", "artifacts": {"type": "var", "name": "result"}} } } } EOI cat > TARGETS <<'EOI' { "full": {"type": "data", "srcs": ["foo.txt", "subdir/foo.txt"]} , "flat": {"type": "data", "srcs": ["foo.txt", "subdir/foo.txt"], "flat": ["YES"]} } EOI bin/tool-under-test build --local-build-root .tool-root -f build.log full 2>&1 echo grep 'DataRuleSpecificErrorMessage' build.log && exit 1 || : grep 'DeBuG-InFoRmAtIoN' build.log && exit 1 || : bin/tool-under-test build --local-build-root .tool-root -f build.log flat 2>&1 && exit 1 || : echo grep 'DataRuleSpecificErrorMessage' build.log grep "DeBuG-InFoRmAtIoN" build.log echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/user-errors/json-errors.sh000077500000000000000000000026371516554100600276520ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e readonly ROOT="${PWD}" readonly JUST="${ROOT}/bin/tool-under-test" readonly LBR="${TEST_TMPDIR}/local-build-root" touch ROOT cat > TARGETS <<'EOF' { "THIS": would be the default target if it where syntactically correct ... EOF "${JUST}" build --local-build-root "${LBR}" 2>msg && exit 1 || : cat msg grep -q 'default target' msg grep -q 'TARGETS' msg grep -q 'line' msg # expect to see a position of the parsing error grep -q '"THIS"' msg # expect to see the last valid token "${JUST}" build --local-build-root "${LBR}" some-target 2>msg && exit 1 || : cat msg # expect the error to happen while trying to analyse the default target grep -q '["@","","","some-target"]' msg grep -q 'line' msg # expect to see a position of the parsing error grep -q '"THIS"' msg # expect to see the last valid token echo OK just-buildsystem-justbuild-b1fb5fa/test/end-to-end/with_remote_test_runner.py000077500000000000000000000113361516554100600300670ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import sys import subprocess import time from typing import Any, Dict Json = Dict[str, Any] time_start: float = time.time() time_stop: float = 0 result: str = "UNKNOWN" stderr: str = "" stdout: str = "" def dump_results() -> None: with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) with open("pwd", "w") as f: f.write("%s\n" % (os.getcwd(), )) def get_remote_execution_address(d: Json) -> str: return "%s:%d" % (d["interface"], int(d["port"])) dump_results() TEMP_DIR = os.path.realpath("scratch") os.makedirs(TEMP_DIR, exist_ok=True) WORK_DIR = os.path.realpath("work") os.makedirs(WORK_DIR, exist_ok=True) REMOTE_DIR = os.path.realpath("remote") os.makedirs(REMOTE_DIR, exist_ok=True) REMOTE_LBR = os.path.join(REMOTE_DIR, "build-root") REMOTE_EXTRA_BINDIR = os.path.join(REMOTE_DIR, "bin") os.makedirs(REMOTE_EXTRA_BINDIR, exist_ok=True) g_REMOTE_EXECUTION_ADDRESS: str = "" compatible = json.loads(sys.argv[1]) custom_remote = json.loads(sys.argv[2]) remote_proc = None if not custom_remote: # start just execute as remote service REMOTE_INFO = os.path.join(REMOTE_DIR, "info.json") if os.path.exists(REMOTE_INFO): print(f"Warning: removing unexpected info file {REMOTE_INFO}") os.remove(REMOTE_INFO) PATH = subprocess.run( ["env", "--", "sh", "-c", "echo -n $PATH"], stdout=subprocess.PIPE, ).stdout.decode('utf-8') remote_cmd = [ "./staged/bin/just", "execute", "-L", json.dumps( ["env", "PATH=" + REMOTE_EXTRA_BINDIR + (":" if PATH else "") + PATH]), "--info-file", REMOTE_INFO, "--local-build-root", REMOTE_LBR, "--log-limit", "6", "--plain-log", ] if compatible: remote_cmd.append("--compatible") remotestdout = open("remotestdout", "w") remotestderr = open("remotestderr", "w") remote_proc = subprocess.Popen( remote_cmd, stdout=remotestdout, stderr=remotestderr, ) timeout: int = 30 while not os.path.exists(REMOTE_INFO): if timeout == 0: result = "FAIL" stdout = "Failed to start execution service" remote_proc.terminate() dump_results() exit(1) timeout -= 1 time.sleep(1) with open(REMOTE_INFO) as f: info = json.load(f) g_REMOTE_EXECUTION_ADDRESS = get_remote_execution_address(info) else: msg = "\nA custom remote service is used, please look at logs there.\n" with open("remotestdout", "w") as f: print(msg, file=f) with open("remotestderr", "w") as f: print(msg, file=f) args = custom_remote.get("args", []) g_REMOTE_EXECUTION_ADDRESS = " ".join( (get_remote_execution_address(custom_remote), *args)) ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR, TMPDIR=TEMP_DIR, REMOTE_EXECUTION_ADDRESS=g_REMOTE_EXECUTION_ADDRESS, REMOTE_BIN=REMOTE_EXTRA_BINDIR) if compatible: ENV["COMPATIBLE"] = "YES" elif "COMPATIBLE" in ENV: del ENV["COMPATIBLE"] for k in ["TLS_CA_CERT", "TLS_CLIENT_CERT", "TLS_CLIENT_KEY"]: if k in ENV: del ENV[k] time_start = time.time() ret = subprocess.run(["sh", "../test.sh"], cwd=WORK_DIR, env=ENV, capture_output=True) time_stop = time.time() result = "PASS" if ret.returncode == 0 else "FAIL" stdout = ret.stdout.decode("utf-8") stderr = ret.stderr.decode("utf-8") if not custom_remote: assert remote_proc remote_proc.terminate() rout, rerr = remote_proc.communicate() dump_results() for f in sys.argv[2:]: keep_file = os.path.join(WORK_DIR, f) if not os.path.exists(keep_file): open(keep_file, "a").close() if result != "PASS": exit(1) just-buildsystem-justbuild-b1fb5fa/test/end-to-end/with_serve_test_runner.py000077500000000000000000000211411516554100600277130ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import subprocess import sys import time from typing import Any, Dict, List Json = Dict[str, Any] time_start: float = time.time() time_stop: float = 0 result: str = "UNKNOWN" stderr: str = "" stdout: str = "" def dump_results() -> None: with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) with open("pwd", "w") as f: f.write("%s\n" % (os.getcwd(), )) def run_cmd(cmd: List[str], *, env: Any = None, stdout: Any = subprocess.DEVNULL, stderr: Any = None, cwd: str): subprocess.run(cmd, cwd=cwd, env=env, stdout=stdout, stderr=stderr) def get_remote_execution_address(d: Json) -> str: return "%s:%d" % (d["interface"], int(d["port"])) dump_results() TEMP_DIR = os.path.abspath(os.path.realpath("scratch")) os.makedirs(TEMP_DIR, exist_ok=True) WORK_DIR = os.path.abspath(os.path.realpath("work")) os.makedirs(WORK_DIR, exist_ok=True) REMOTE_DIR = os.path.abspath(os.path.realpath("remote")) os.makedirs(REMOTE_DIR, exist_ok=True) REMOTE_LBR = os.path.join(REMOTE_DIR, "build-root") g_REMOTE_EXECUTION_ADDRESS: str = "" SERVE_DIR = os.path.abspath(os.path.realpath("serve")) os.makedirs(SERVE_DIR, exist_ok=True) SERVE_LBR = os.path.join(SERVE_DIR, "build-root") compatible = json.loads(sys.argv[1]) standalone_serve = json.loads(sys.argv[2]) remote_proc = None PATH = subprocess.run( ["env", "--", "sh", "-c", "echo -n $PATH"], stdout=subprocess.PIPE, ).stdout.decode('utf-8') remotestdout = open("remotestdout", "w") remotestderr = open("remotestderr", "w") if not standalone_serve: # start just execute as remote service REMOTE_INFO = os.path.join(REMOTE_DIR, "remote-info.json") if os.path.exists(REMOTE_INFO): print(f"Warning: removing unexpected info file {REMOTE_INFO}") os.remove(REMOTE_INFO) remote_cmd = [ "./staged/bin/just", "execute", "-L", json.dumps(["env", "PATH=" + PATH]), "--info-file", REMOTE_INFO, "--local-build-root", REMOTE_LBR, "--log-limit", "6", "--plain-log", ] if compatible: remote_cmd.append("--compatible") remote_proc = subprocess.Popen( remote_cmd, stdout=remotestdout, stderr=remotestderr, ) while not os.path.exists(REMOTE_INFO): time.sleep(1) with open(REMOTE_INFO) as f: info = json.load(f) g_REMOTE_EXECUTION_ADDRESS = get_remote_execution_address(info) # start just serve service SERVE_INFO = os.path.join(SERVE_DIR, "serve-info.json") SERVE_CONFIG_FILE = os.path.join(SERVE_DIR, "serve.json") serve_config: Json = {} if standalone_serve: serve_config = { "local build root": { "root": "system", "path": SERVE_LBR }, "logging": { "limit": 6, "plain": True }, "execution endpoint": { "compatible": compatible }, "remote service": { "info file": { "root": "system", "path": SERVE_INFO } }, "build": { "local launcher": ["env", "PATH=" + PATH] }, } else: serve_config = { "local build root": { "root": "system", "path": SERVE_LBR }, "logging": { "limit": 6, "plain": True }, "execution endpoint": { "address": g_REMOTE_EXECUTION_ADDRESS, "compatible": compatible }, "remote service": { "info file": { "root": "system", "path": SERVE_INFO } }, } repositories: List[Dict[str, str]] = [] # list of location objects repos_env: Dict[str, str] = {} REPOS_DIR = os.path.realpath("repos") os.makedirs(REPOS_DIR, exist_ok=True) DATA_DIR = os.path.realpath("data") os.makedirs(DATA_DIR, exist_ok=True) GIT_NOBODY_ENV: Dict[str, str] = { "GIT_AUTHOR_DATE": "1970-01-01T00:00Z", "GIT_AUTHOR_NAME": "Nobody", "GIT_AUTHOR_EMAIL": "nobody@example.org", "GIT_COMMITTER_DATE": "1970-01-01T00:00Z", "GIT_COMMITTER_NAME": "Nobody", "GIT_COMMITTER_EMAIL": "nobody@example.org", "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", } count = 0 repo_data = sorted(os.listdir("data")) for repo in repo_data: target = os.path.join(REPOS_DIR, repo) shutil.copytree( os.path.join(DATA_DIR, repo), target, ) run_cmd( ["git", "init"], cwd=target, env=dict(os.environ, **GIT_NOBODY_ENV), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) run_cmd( ["git", "add", "-f", "."], cwd=target, env=dict(os.environ, **GIT_NOBODY_ENV), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) run_cmd( ["git", "commit", "-m", "Content of %s" % (target, )], cwd=target, env=dict(os.environ, **GIT_NOBODY_ENV), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) repositories.append({"root": "system", "path": target}) repos_env["COMMIT_%d" % count] = subprocess.run( ["git", "log", "-n", "1", "--format=%H"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, cwd=target).stdout.decode('utf-8').strip() repos_env["TREE_%d" % count] = subprocess.run( ["git", "log", "-n", "1", "--format=%T"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, cwd=target).stdout.decode('utf-8').strip() # increase counter count += 1 serve_config["repositories"] = repositories with open(SERVE_CONFIG_FILE, "w") as f: json.dump(serve_config, f) servestdout = open("servestdout", "w") servestderr = open("servestderr", "w") serve_proc = subprocess.Popen( ["./staged/bin/just", "serve", SERVE_CONFIG_FILE], stdout=servestdout, stderr=servestderr, ) timeout: int = 30 while not os.path.exists(SERVE_INFO): if timeout == 0: result = "FAIL" stdout = "Failed to start serve service" serve_proc.terminate() dump_results() exit(1) timeout -= 1 time.sleep(1) with open(SERVE_INFO) as f: serve_info = json.load(f) SERVE_ADDRESS = get_remote_execution_address(serve_info) # run the actual test ENV = dict( os.environ, TEST_TMPDIR=TEMP_DIR, TMPDIR=TEMP_DIR, REMOTE_EXECUTION_ADDRESS=(g_REMOTE_EXECUTION_ADDRESS if not standalone_serve else SERVE_ADDRESS), REMOTE_LBR=(REMOTE_LBR if not standalone_serve else SERVE_LBR), SERVE=SERVE_ADDRESS, SERVE_LBR=SERVE_LBR, # expose the serve build root to the test env **repos_env) if standalone_serve: ENV["STANDALONE_SERVE"] = "YES" elif "STANDALONE_SERVE" in ENV: del ENV["STANDALONE_SERVE"] if compatible: ENV["COMPATIBLE"] = "YES" elif "COMPATIBLE" in ENV: del ENV["COMPATIBLE"] for k in ["TLS_CA_CERT", "TLS_CLIENT_CERT", "TLS_CLIENT_KEY"]: if k in ENV: del ENV[k] time_start = time.time() ret = subprocess.run(["sh", "../test.sh"], cwd=WORK_DIR, env=ENV, capture_output=True) time_stop = time.time() result = "PASS" if ret.returncode == 0 else "FAIL" stdout = ret.stdout.decode("utf-8") stderr = ret.stderr.decode("utf-8") if not standalone_serve: assert remote_proc remote_proc.terminate() rout, rerr = remote_proc.communicate() assert serve_proc serve_proc.terminate() sout, serr = serve_proc.communicate() dump_results() for f in sys.argv[3:]: keep_file = os.path.join(WORK_DIR, f) if not os.path.exists(keep_file): open(keep_file, "a").close() if result != "PASS": exit(1) just-buildsystem-justbuild-b1fb5fa/test/main.cpp000066400000000000000000000025121516554100600222260ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "catch2/catch_session.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/storage/file_chunker.hpp" #include "test/utils/logging/log_config.hpp" auto main(int argc, char* argv[]) -> int { ConfigureLogging(); /** * The current implementation of libgit2 uses pthread_key_t incorrectly * on POSIX systems to handle thread-specific data, which requires us to * explicitly make sure the main thread is the first one to call * git_libgit2_init. Future versions of libgit2 will hopefully fix this. */ GitContext::Create(); // Initialize random content of the file chunker's map. FileChunker::Initialize(); return Catch::Session().run(argc, argv); } just-buildsystem-justbuild-b1fb5fa/test/other_tools/000077500000000000000000000000001516554100600231375ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/other_tools/TARGETS000066400000000000000000000003241516554100600241720ustar00rootroot00000000000000{ "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["other_tools"] , "deps": [ ["./", "git_operations", "TESTS"] , ["./", "just_mr", "TESTS"] , ["./", "utils", "TESTS"] ] } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/000077500000000000000000000000001516554100600261655ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/TARGETS000066400000000000000000000057421516554100600272310ustar00rootroot00000000000000{ "critical_git_ops_test_install": { "type": ["@", "rules", "CC", "binary"] , "tainted": ["test"] , "name": ["critical_git_ops_test"] , "srcs": ["critical_git_ops.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/common", "ids"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/multithreading", "task_system"] , ["@", "src", "src/other_tools/git_operations", "git_ops_types"] , ["@", "src", "src/other_tools/ops_maps", "critical_git_op_map"] , ["", "catch-main"] , ["utils", "shell_quoting"] ] , "stage": ["test", "other_tools", "git_operations"] } , "critical_git_ops_mp": { "type": ["@", "rules", "shell/test", "script"] , "name": ["critical_git_ops_mp"] , "test": ["critical_git_ops_mp.sh"] , "deps": ["critical_git_ops_test_install", ["buildtool/file_system", "test_data"]] } , "git_repo_remote": { "type": ["@", "rules", "CC/test", "test"] , "name": ["git_repo_remote"] , "srcs": ["git_repo_remote.test.cpp"] , "data": [["buildtool/file_system", "test_data"]] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "fmt", "", "fmt"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/buildtool/file_system", "git_cas"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/other_tools/git_operations", "git_repo_remote"] , ["@", "src", "src/utils/cpp", "atomic"] , ["", "catch-main"] , ["utils", "shell_quoting"] , ["utils", "test_storage_config"] ] , "stage": ["test", "other_tools", "git_operations"] } , "git_config_run": { "type": ["@", "rules", "CC", "binary"] , "tainted": ["test"] , "name": ["git_config_run_test"] , "srcs": ["git_config_run.test.cpp"] , "private-deps": [ ["@", "src", "", "libgit2"] , ["@", "src", "src/buildtool/file_system", "git_context"] , ["@", "src", "src/buildtool/file_system", "git_utils"] , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/other_tools/git_operations", "git_config_settings"] , ["utils", "log_config"] ] , "stage": ["src"] } , "git_config_ssl": { "type": ["@", "rules", "shell/test", "script"] , "tainted": ["test"] , "name": ["git_config_ssl"] , "test": ["git_config_ssl.sh"] , "deps": ["git_config_run"] } , "git_config_proxy": { "type": ["@", "rules", "shell/test", "script"] , "tainted": ["test"] , "name": ["git_config_proxy"] , "test": ["git_config_proxy.sh"] , "deps": ["git_config_run"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["git_operations"] , "deps": [ "critical_git_ops_mp" , "git_config_proxy" , "git_config_ssl" , "git_repo_remote" ] } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/critical_git_ops.test.cpp000066400000000000000000000246371516554100600332010ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include // std::find #include #include #include // std::system #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/other_tools/git_operations/git_ops_types.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kRootCommit = std::string{"3ecce3f5b19ad7941c6354d65d841590662f33ef"}; auto const kBazSymId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"}; } // namespace /// \brief TestUtils that accounts for multi-process calls. /// Ensures the git clone only happens once per path. /// Can also create process-unique paths. class TestUtilsMP { public: [[nodiscard]] static auto GetUniqueTestDir() noexcept -> std::optional { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return CreateUniquePath(tmp_dir); } return CreateUniquePath(FileSystemManager::GetCurrentDirectory() / "test/other_tools"); } [[nodiscard]] static auto GetRepoPath( std::filesystem::path const& prefix) noexcept -> std::filesystem::path { return prefix / "test_git_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); } [[nodiscard]] static auto GetRepoPathUnique( std::filesystem::path const& prefix) noexcept -> std::filesystem::path { return prefix / ("test_git_repo." + CreateProcessUniqueId().value()) / std::filesystem::path{std::to_string(counter++)}.filename(); } // The checkout will make the content available, as well as the HEAD ref [[nodiscard]] static auto CreateTestRepoWithCheckout( std::filesystem::path const& prefix, bool is_bare = false) noexcept -> std::optional { auto repo_path = CreateTestRepo(prefix, is_bare); REQUIRE(repo_path); auto cmd = fmt::format("git --git-dir={} --work-tree={} checkout master", QuoteForShell(is_bare ? repo_path->string() : (*repo_path / ".git").string()), QuoteForShell(repo_path->string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] static auto CreateTestRepo( std::filesystem::path const& prefix, bool is_bare = false) noexcept -> std::optional { auto repo_path = GetRepoPath(prefix); std::optional result = std::nullopt; // only do work if another process hasn't already been here if (not FileSystemManager::Exists(repo_path)) { auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { result = repo_path; } } else { result = repo_path; } return result; } private: // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static inline std::atomic counter = 0; }; TEST_CASE("Critical git operations", "[critical_git_op_map]") { // setup the repos needed auto prefix = TestUtilsMP::GetUniqueTestDir(); REQUIRE(prefix); auto testdir = *prefix / "test_git_repo"; REQUIRE(FileSystemManager::CreateDirectory(testdir)); // create the remote for the fetch ops auto remote_repo_path = TestUtilsMP::CreateTestRepoWithCheckout(*prefix); REQUIRE(remote_repo_path); // create the target paths for the various critical ops // IMPORTANT! For non-init critical ops the paths need to exist already! // 0. Initial commit -> needs a path containing some files // This has to be process unique, as the commit will fail otherwise! auto path_init_commit = TestUtilsMP::GetRepoPathUnique(*prefix); REQUIRE(FileSystemManager::WriteFile( "test no 1", path_init_commit / "test1.txt", true)); REQUIRE(FileSystemManager::WriteFile( "test no 2", path_init_commit / "test2.txt", true)); // 1 & 2. Initializing repos -> need only the paths auto path_init_bare = TestUtilsMP::GetRepoPath(*prefix); auto path_init_non_bare = TestUtilsMP::GetRepoPath(*prefix); // 3. Tag a commit -> needs a repo with a commit auto path_keep_tag = TestUtilsMP::CreateTestRepo(*prefix, true); REQUIRE(path_keep_tag); // 4. Get head commit -> needs a repo with HEAD ref available auto path_get_head_id = TestUtilsMP::CreateTestRepoWithCheckout(*prefix); REQUIRE(path_get_head_id); // 5. Tag a tree -> needs a repo with a tree auto path_keep_tree = TestUtilsMP::CreateTestRepo(*prefix, true); REQUIRE(path_keep_tree); // create the map auto crit_op_guard = std::make_shared(); auto crit_op_map = CreateCriticalGitOpMap(crit_op_guard); // Add ops to the map. None should throw, as repeating the same operation // should retrieve the value from the map, not call the operation again. // helper lists constexpr auto kNumMethods = 6; std::vector ops_all(kNumMethods); // indices of all ops tested std::iota(ops_all.begin(), ops_all.end(), 0); const std::vector ops_with_result{ 0, 4}; // indices of ops that return a non-empty string // Add to the map all ops multiple times constexpr auto kRepeats = 3; for ([[maybe_unused]] auto k = kRepeats; k > 0; --k) { auto error = false; auto error_msg = std::string("NONE"); { TaskSystem ts; for ([[maybe_unused]] auto j = kRepeats; j > 0; --j) { crit_op_map.ConsumeAfterKeysReady( &ts, {GitOpKey{.params = { path_init_commit, // target_path "", // git_hash "Init commit", // message path_init_commit // source_path }, .op_type = GitOpType::INITIAL_COMMIT}, GitOpKey{.params = { path_init_bare, // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path true // init_bare }, .op_type = GitOpType::ENSURE_INIT}, GitOpKey{.params = { path_init_non_bare, // target_path "", // git_hash std::nullopt, // message std::nullopt, // source_path false // init_bare }, .op_type = GitOpType::ENSURE_INIT}, GitOpKey{.params = { *path_keep_tag, // target_path kRootCommit, // git_hash "keep-commit" // message }, .op_type = GitOpType::KEEP_TAG}, GitOpKey{.params = { *path_get_head_id, // target_path "", // git_hash }, .op_type = GitOpType::GET_HEAD_ID}, GitOpKey{.params = { *path_keep_tree, // target_path kBazSymId, // git_hash "keep-tree" // message }, .op_type = GitOpType::KEEP_TREE}}, [&ops_all, &ops_with_result](auto const& values) { // check operations for (std::size_t const& i : ops_all) { auto res = *values[i]; REQUIRE(res.git_cas); REQUIRE(res.result); if (std::find(ops_with_result.begin(), ops_with_result.end(), i) != ops_with_result.end()) { CHECK(not res.result->empty()); } } }, [&error, &error_msg](std::string const& msg, bool /*unused*/) { error = true; error_msg = msg; }); } } CHECK_FALSE(error); CHECK(error_msg == "NONE"); } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/critical_git_ops_mp.sh000066400000000000000000000024371516554100600325410ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu # Run the critical git ops test multiple times from different processes echo "running critical git ops test from 4 processes" error=false test/other_tools/git_operations/critical_git_ops_test & res1=$! test/other_tools/git_operations/critical_git_ops_test & res2=$! test/other_tools/git_operations/critical_git_ops_test & res3=$! test/other_tools/git_operations/critical_git_ops_test & res4=$! wait $res1 if [ $? -ne 0 ]; then error=true fi wait $res2 if [ $? -ne 0 ]; then error=true fi wait $res3 if [ $? -ne 0 ]; then error=true fi wait $res4 if [ $? -ne 0 ]; then error=true fi # if one fails, set fail overall if [ $error = true ]; then exit 1 fi echo "done" just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/git_config_proxy.sh000066400000000000000000000167061516554100600321040ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ### # Run Git config proxy settings retrieval tests for various scenarios # # Expected Git config (relative) file path: gitconfig ## readonly ROOT=`pwd` readonly TESTEXEC="${ROOT}/src/git_config_run_test" # ensure clean env for this type of test unset all_proxy unset ALL_PROXY unset no_proxy unset NO_PROXY unset http_proxy unset https_proxy unset HTTPS_PROXY # Set up the test scenarios to be run in parallel k=1 # scenarios counter echo echo "Scenario $k: default (clean env and missing config)" k=$((k+1)) envcmd="" url="https://192.0.2.1" result="" $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy from all_proxy envariable" k=$((k+1)) envcmd="all_proxy=198.51.100.1:50000" url="https://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy from ALL_PROXY envariable" k=$((k+1)) envcmd="ALL_PROXY=198.51.100.1:50000" url="https://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: lowercase all_proxy envariable priority" k=$((k+1)) envcmd="all_proxy=198.51.100.1:50000 ALL_PROXY=192.0.2.1:50505" url="https://example.com" # scheme does not match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy from http_proxy" k=$((k+1)) envcmd="http_proxy=198.51.100.1:50000" url="http://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy from https_proxy" k=$((k+1)) envcmd="https_proxy=198.51.100.1:50000" url="https://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy from HTTPS_PROXY" k=$((k+1)) envcmd="HTTPS_PROXY=198.51.100.1:50000" url="https://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: scheme mismatch in envariable" k=$((k+1)) envcmd="https_proxy=198.51.100.1:50000" url="http://example.com" # scheme does not match envariable result="" $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: lowercase http_proxy envariable priority" k=$((k+1)) envcmd="https_proxy=198.51.100.1:50000 HTTPS_PROXY=192.0.2.1:50505" url="https://example.com" # scheme does not match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: specific and generic envariables priority" k=$((k+1)) envcmd="http_proxy=198.51.100.1:50000 all_proxy=192.0.2.1:50505" url="http://example.com" # scheme must match envariable result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) eval "${envcmd} ${TESTEXEC} proxy ${url} ${result}" cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: proxy via http.proxy config entry" k=$((k+1)) envcmd="" url="https://example.com" result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) cat > gitconfig <.proxy config entry" k=$((k+1)) envcmd="" url="https://example.com" result="http://198.51.100.1:50000/" # proxy default scheme is http $( cd $(mktemp -d) cat > gitconfig < gitconfig < gitconfig <.proxy config entry" k=$((k+1)) envcmd="" url="https://example.com" result="" # no proxy $( cd $(mktemp -d) cat > gitconfig < gitconfig < gitconfig < gitconfig < gitconfig < #include #include #include #include #include #include #include #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/other_tools/git_operations/git_config_settings.hpp" #include "test/utils/logging/log_config.hpp" extern "C" { #include } namespace { using anon_logger_t = std::function; auto const kGitConfigPath = std::filesystem::path{"gitconfig"}; } // namespace /// \brief Expects 2 mandatory arguments: /// 1. the test type: "SSL" | "proxy" /// 2. the remote URL to test /// The third argument gives the expected result to check against: /// - for type "SSL": anything (check SSL) | missing arg (passthrough) /// - for type "proxy": proxy string (exact match) | missing arg (no proxy) auto main(int argc, char* argv[]) -> int { try { ConfigureLogging(); // start a git context, needed to read in the config file GitContext::Create(); // handle args if (argc < 3) { Logger::Log(LogLevel::Error, "Expected at least 3 args, but found {}", argc); return 1; } auto args = std::span(argv, static_cast(argc)); std::string test_type{args[1]}; // type of test std::string test_url{args[2]}; // remote URL to test // setup dummy logger auto logger = std::make_shared( []([[maybe_unused]] auto const& msg, [[maybe_unused]] bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); // read in the git config file git_config* cfg_ptr{nullptr}; if (git_config_open_ondisk(&cfg_ptr, kGitConfigPath.c_str()) != 0) { Logger::Log(LogLevel::Error, "Open git config on disk failed"); return 1; } auto cfg = std::shared_ptr(cfg_ptr, config_closer); // run the method for given type if (test_type == "SSL") { auto callback = GitConfigSettings::GetSSLCallback(cfg, test_url, logger); if (not callback) { Logger::Log(LogLevel::Error, "Null SSL callback"); return 1; } // check expected result auto expected_result = static_cast(argc >= 4); auto actual_result = callback.value()(nullptr, 0, nullptr, nullptr); if (actual_result != expected_result) { Logger::Log(LogLevel::Error, "Expected test result {}, but obtained {}", expected_result, actual_result); return 1; } } else if (test_type == "proxy") { auto proxy_info = GitConfigSettings::GetProxySettings(cfg, test_url, logger); if (not proxy_info) { Logger::Log(LogLevel::Error, "Missing proxy_info"); return 1; } // check expected result auto expected_result = (argc >= 4) ? std::make_optional(args[3]) : std::nullopt; auto actual_result = proxy_info.value(); if (actual_result != expected_result) { Logger::Log(LogLevel::Error, "Expected test result {}, but obtained {}", expected_result ? *expected_result : "nullopt", actual_result ? *actual_result : "nullopt"); return 1; } } else { Logger::Log(LogLevel::Error, R"(Expected test type {"SSL"|"proxy"}, but found {})", test_type); return 1; } } catch (std::exception const& ex) { Logger::Log( LogLevel::Error, "Git config run test failed with:\n{}", ex.what()); return 1; } return 0; } just-buildsystem-justbuild-b1fb5fa/test/other_tools/git_operations/git_config_ssl.sh000066400000000000000000000042601516554100600315140ustar00rootroot00000000000000#!/bin/sh # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu ### # Run Git config SSL settings retrieval tests for various scenarios # # Expected Git config (relative) file path: gitconfig # # ## readonly ROOT=`pwd` readonly TESTEXEC="${ROOT}/src/git_config_run_test" # ensure clean env for this type of test unset GIT_SSL_NO_VERIFY # run the test scenarios k=1 # scenarios counter echo echo "Scenario $k: missing gitconfig" k=$((k+1)) $( cd $(mktemp -d) ${TESTEXEC} SSL https://192.0.2.1 1 cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: empty gitconfig" k=$((k+1)) $( cd $(mktemp -d) touch gitconfig ${TESTEXEC} SSL https://example.com 1 cd ${ROOT} ) echo "PASSED" echo echo "Scenario $k: disable SSL check via http.sslVerify" k=$((k+1)) $( cd $(mktemp -d) cat > gitconfig <.sslVerify" k=$((k+1)) $( cd $(mktemp -d) cat > gitconfig < gitconfig < gitconfig < #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/atomic.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; auto const kRootCommit = std::string{"3ecce3f5b19ad7941c6354d65d841590662f33ef"}; auto const kRootId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; } // namespace class TestUtils { public: [[nodiscard]] static auto GetTestDir() noexcept -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/other_tools"; } [[nodiscard]] static auto GetRepoPath() noexcept -> std::filesystem::path { return GetTestDir() / "test_git_repo" / std::filesystem::path{std::to_string(counter++)}.filename(); } // The checkout will make the content available, as well as the HEAD ref [[nodiscard]] static auto CreateTestRepoWithCheckout( bool is_bare = false) noexcept -> std::optional { auto repo_path = CreateTestRepo(is_bare); REQUIRE(repo_path); auto cmd = fmt::format("git --git-dir={} --work-tree={} checkout master", QuoteForShell(is_bare ? repo_path->string() : (*repo_path / ".git").string()), QuoteForShell(repo_path->string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } [[nodiscard]] static auto CreateTestRepo(bool is_bare = false) noexcept -> std::optional { auto repo_path = GetRepoPath(); auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(kBundlePath), QuoteForShell(repo_path.string())); if (std::system(cmd.c_str()) == 0) { return repo_path; } return std::nullopt; } private: // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static inline std::atomic counter = 0; }; /* NOTE: GitRepoRemote inherits from GitRepo all the methods relating to non-remote Git operations. Those methods are already accounted for int he GitRepo tests, therefore they are skipped here to avoid superfluous work. */ TEST_CASE("Open extended Git repo", "[git_repo_remote]") { SECTION("Fake bare repository") { auto repo_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); // same odb, same GitCAS CHECK(repo->IsRepoFake()); } SECTION("Fake non-bare repository") { auto repo_path = TestUtils::CreateTestRepo(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); // same odb, same GitCAS CHECK(repo->IsRepoFake()); } SECTION("Real bare repository") { auto repo_path = TestUtils::CreateTestRepo(true); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(*repo_path); REQUIRE(repo); CHECK_FALSE(repo->GetGitCAS() == cas); // same odb, different GitCAS CHECK_FALSE(repo->IsRepoFake()); } SECTION("Real non-bare repository") { auto repo_path = TestUtils::CreateTestRepo(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(*repo_path); REQUIRE(repo); CHECK_FALSE(repo->GetGitCAS() == cas); // same odb, different GitCAS CHECK_FALSE(repo->IsRepoFake()); } SECTION("Non-existing repository") { auto repo = GitRepoRemote::Open("does_not_exist"); REQUIRE(repo == std::nullopt); } SECTION("Initialize and open bare repository") { auto repo_path = TestUtils::GetRepoPath(); auto repo = GitRepoRemote::InitAndOpen(repo_path, /*is_bare=*/true); REQUIRE(repo); CHECK_FALSE(repo->IsRepoFake()); } SECTION("Real non-bare repository with checkout") { auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(cas); REQUIRE(repo); CHECK(repo->GetGitCAS() == cas); CHECK(repo->IsRepoFake()); } } TEST_CASE("Single-threaded real repository remote operations", "[git_repo_remote]") { auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); SECTION("Get commit id from remote") { // make bare real repo to call remote ls from auto path_remote_ls_bare = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(path_remote_ls_bare); auto repo_remote_ls_bare = GitRepoRemote::Open(*path_remote_ls_bare); REQUIRE(repo_remote_ls_bare); // remote ls auto remote_commit = repo_remote_ls_bare->GetCommitFromRemote( nullptr, *repo_path, "master", logger); REQUIRE(remote_commit); CHECK(*remote_commit == kRootCommit); } SECTION("Fetch with base refspecs from remote") { // make bare real repo to fetch into auto path_fetch_all_bare = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(path_fetch_all_bare); auto repo_fetch_all_bare = GitRepoRemote::Open(*path_fetch_all_bare); // fetch CHECK(repo_fetch_all_bare->FetchFromRemote( nullptr, *repo_path, std::nullopt, logger)); } SECTION("Fetch branch from remote") { // make bare real repo to fetch into auto path_fetch_branch_bare = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(path_fetch_branch_bare); auto repo_fetch_branch_bare = GitRepoRemote::Open(*path_fetch_branch_bare); REQUIRE(repo_fetch_branch_bare); // fetch CHECK(repo_fetch_branch_bare->FetchFromRemote( nullptr, *repo_path, "master", logger)); } } TEST_CASE("Single-threaded fake repository operations", "[git_repo_remote]") { auto const storage_config = TestStorageConfig::Create(); auto repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(repo_path); auto cas = GitCAS::Open(*repo_path); REQUIRE(cas); auto repo = GitRepoRemote::Open(cas); REQUIRE(repo); REQUIRE(repo->GetGitCAS() == cas); REQUIRE(repo->IsRepoFake()); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); SECTION("Fetch objects from remote via temporary repository") { SECTION("Fetch all into repository") { // set repo to fetch into auto path_fetch_all = TestUtils::GetRepoPath(); auto repo_fetch_all = GitRepoRemote::InitAndOpen(path_fetch_all, /*is_bare=*/true); REQUIRE(repo_fetch_all); // check commit is not there before fetch CHECK_FALSE( *repo_fetch_all->CheckCommitExists(kRootCommit, logger)); // fetch all with base refspecs REQUIRE(repo_fetch_all->FetchViaTmpRepo(storage_config.Get(), *repo_path, std::nullopt, {}, "git", {}, logger)); // check commit is there after fetch CHECK(*repo_fetch_all->CheckCommitExists(kRootCommit, logger)); } SECTION("Fetch with refspec into repository") { // set repo to fetch into auto path_fetch_refspec = TestUtils::GetRepoPath(); auto repo_fetch_refspec = GitRepoRemote::InitAndOpen( path_fetch_refspec, /*is_bare=*/true); REQUIRE(repo_fetch_refspec); // check commit is not there before fetch CHECK_FALSE( *repo_fetch_refspec->CheckCommitExists(kRootCommit, logger)); // fetch all REQUIRE(repo_fetch_refspec->FetchViaTmpRepo(storage_config.Get(), *repo_path, "master", {}, "git", {}, logger)); // check commit is there after fetch CHECK(*repo_fetch_refspec->CheckCommitExists(kRootCommit, logger)); } } SECTION("Update commit from remote via temporary repository") { auto path_commit_upd = TestUtils::GetRepoPath(); auto repo_commit_upd = GitRepoRemote::InitAndOpen(path_commit_upd, /*is_bare=*/true); REQUIRE(repo_commit_upd); // do remote ls auto fetched_commit = repo_commit_upd->UpdateCommitViaTmpRepo( storage_config.Get(), *repo_path, "master", {}, "git", {}, logger); REQUIRE(fetched_commit); CHECK(*fetched_commit == kRootCommit); } } TEST_CASE("Multi-threaded fake repository operations", "[git_repo_remote]") { auto const storage_config = TestStorageConfig::Create(); /* Test all fake repository operations while being done in parallel. They are supposed to be thread-safe, so no conflicts should exist. */ // define remote, for ops that need it auto remote_repo_path = TestUtils::CreateTestRepoWithCheckout(); REQUIRE(remote_repo_path); auto remote_cas = GitCAS::Open(*remote_repo_path); REQUIRE(remote_cas); auto remote_repo = GitRepoRemote::Open(remote_cas); REQUIRE(remote_repo); REQUIRE(remote_repo->GetGitCAS() == remote_cas); REQUIRE(remote_repo->IsRepoFake()); // setup dummy logger auto logger = std::make_shared( [](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Progress, std::string(msg)); }); // setup threading constexpr auto kNumThreads = 100; atomic starting_signal{false}; std::vector threads{}; threads.reserve(kNumThreads); // define target repo, from which fetch ops will be initiated auto target_repo_path = TestUtils::GetRepoPath(); auto target_repo = GitRepoRemote::InitAndOpen(target_repo_path, /*is_bare=*/true); REQUIRE(target_repo); SECTION("Fetching into same repository from remote") { constexpr int kNumCases = 4; for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&storage_config, &target_repo, &remote_repo_path, &logger, &starting_signal](int tid) { starting_signal.wait(false); // cases based on thread number switch (tid % kNumCases) { case 0: { auto result_containing = target_repo->CheckCommitExists(kRootCommit, logger); CHECK(result_containing); // check it returns // something } break; case 1: { // fetch with base refspecs CHECK(target_repo->FetchViaTmpRepo( storage_config.Get(), *remote_repo_path, std::nullopt, {}, "git", {}, logger)); } break; case 2: { // fetch specific branch CHECK(target_repo->FetchViaTmpRepo( storage_config.Get(), *remote_repo_path, "master", {}, "git", {}, logger)); } break; case 3: { // do remote ls auto fetched_commit = target_repo->UpdateCommitViaTmpRepo( storage_config.Get(), *remote_repo_path, "master", {}, "git", {}, logger); REQUIRE(fetched_commit); CHECK(*fetched_commit == kRootCommit); } break; default: REQUIRE(false); } }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/just_mr/000077500000000000000000000000001516554100600246225ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/other_tools/just_mr/TARGETS000066400000000000000000000017151516554100600256620ustar00rootroot00000000000000{ "rc_merge": { "type": ["@", "rules", "CC/test", "test"] , "name": ["rc_merge"] , "srcs": ["rc_merge.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/build_engine/expression", "expression"] , [ "@" , "src" , "src/buildtool/build_engine/expression" , "expression_ptr_interface" ] , ["@", "src", "src/other_tools/just_mr", "rc_merge"] , ["", "catch-main"] ] , "stage": ["test", "other_tools", "just_mr"] } , "mirrors": { "type": ["@", "rules", "CC/test", "test"] , "name": ["mirrors"] , "srcs": ["mirrors.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/other_tools/just_mr", "mirrors"] , ["", "catch-main"] ] , "stage": ["test", "other_tools", "just_mr"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["just_mr"] , "deps": ["mirrors", "rc_merge"] } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/just_mr/mirrors.test.cpp000066400000000000000000000041601516554100600300020ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/mirrors.hpp" #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_all.hpp" TEST_CASE("SortByHostname") { // setup inputs auto mirrors = std::vector({"file://foo/bar", "https://keep.me/here", "https://example.com:420/foo bar", "./testing", "https://example.com:420/foo baz", "https://keep.me/second", "http://user@bar.baz/foobar"}); auto hostnames = std::vector({"bar.baz", "example.com", "bar.baz"}); // compute ordered mirrors auto ordered = MirrorsUtils::SortByHostname(mirrors, hostnames); // compare with expected, Equals() honors order auto expected = std::vector({"http://user@bar.baz/foobar", "https://example.com:420/foo bar", "https://example.com:420/foo baz", "file://foo/bar", "https://keep.me/here", "./testing", "https://keep.me/second"}); CHECK_THAT(ordered, Catch::Matchers::Equals(expected)); } just-buildsystem-justbuild-b1fb5fa/test/other_tools/just_mr/rc_merge.test.cpp000066400000000000000000000055741516554100600301020ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/rc_merge.hpp" #include "catch2/catch_test_macros.hpp" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" TEST_CASE("Simple field") { auto conf = Configuration{Expression::FromJson(R"( { "log limit": 4 , "git": {"root": "system", "path": "usr/bin/git"} })"_json)}; auto delta = Configuration{Expression::FromJson(R"( { "log limit": 5 })"_json)}; auto merged = MergeMRRC(conf, delta); CHECK(merged["log limit"] == Expression::FromJson("5"_json)); CHECK(merged["git"] == Expression::FromJson( R"({"root": "system", "path": "usr/bin/git"})"_json)); } TEST_CASE("accumulating") { auto conf = Configuration{Expression::FromJson(R"( {"distdirs": [{"root": "home", "path": ".distfiles"}]} )"_json)}; auto delta = Configuration{Expression::FromJson(R"( {"distdirs": [{"root": "workspace", "path": "third_party"}]} )"_json)}; auto merged = MergeMRRC(conf, delta); CHECK(merged["distdirs"] == Expression::FromJson(R"( [ {"root": "workspace", "path": "third_party"} , {"root": "home", "path": ".distfiles"} ] )"_json)); } TEST_CASE("local-merge") { auto conf = Configuration{Expression::FromJson(R"( {"just args": {"build": ["-J", "8"], "install": ["-J", "8", "--remember"]} ,"invocation log": {"directory": {"root": "system" , "path": "/var/log/just-mr"}} } )"_json)}; auto delta = Configuration{Expression::FromJson(R"( {"just args": {"build": ["-J", "128"], "install-cas": ["--remember"]} ,"invocation log": {"project id": "unicorn"} } )"_json)}; auto merged = MergeMRRC(conf, delta); CHECK(merged["just args"] == Expression::FromJson(R"( { "build": ["-J", "128"] , "install-cas": ["--remember"] , "install": ["-J", "8", "--remember"] } )"_json)); CHECK(merged["invocation log"] == Expression::FromJson(R"( { "directory":{"root": "system" , "path": "/var/log/just-mr"} , "project id": "unicorn" } )"_json)); } just-buildsystem-justbuild-b1fb5fa/test/other_tools/utils/000077500000000000000000000000001516554100600242775ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/other_tools/utils/TARGETS000066400000000000000000000021611516554100600253330ustar00rootroot00000000000000{ "curl_usage_install": { "type": ["@", "rules", "CC", "binary"] , "tainted": ["test"] , "name": ["curl_usage_install"] , "srcs": ["curl_usage.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/other_tools/utils", "curl_context"] , ["@", "src", "src/other_tools/utils", "curl_easy_handle"] , ["", "catch-main"] ] , "stage": ["test", "other_tools", "utils"] } , "curl_usage": { "type": ["@", "rules", "shell/test", "script"] , "name": ["curl_usage"] , "test": ["curl_usage_test.sh"] , "deps": ["curl_usage_install", ["utils", "test_utils_install"]] } , "curl_url": { "type": ["@", "rules", "CC/test", "test"] , "name": ["curl_url"] , "srcs": ["curl_url.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/other_tools/utils", "curl_url_handle"] , ["", "catch-main"] ] , "stage": ["test", "other_tools", "utils"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["utils"] , "deps": ["curl_url", "curl_usage"] } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/utils/curl_url.test.cpp000066400000000000000000000272661516554100600276250ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/other_tools/utils/curl_url_handle.hpp" TEST_CASE("Curl URL handle basics", "[curl_url_handle_basics]") { SECTION("Parse URL") { // full syntax check auto url_h_full = CurlURLHandle::Create( "https://user:pass@example.com:50000/some/" "pa.th?what=what&who=who#fragment"); CHECK(url_h_full); CHECK(*url_h_full); // bare bone syntax check auto url_h_thin = CurlURLHandle::Create("http://example.com"); CHECK(url_h_thin); CHECK(*url_h_thin); // double dots in hostname auto url_h_double_dots = CurlURLHandle::Create("http://..example..com"); CHECK(url_h_double_dots); CHECK(*url_h_double_dots); // fail check auto url_h_fail = CurlURLHandle::Create("file://foo:50505"); CHECK(url_h_fail); CHECK(not *url_h_fail); } SECTION("Get URL") { auto url_h = CurlURLHandle::Create("http://example.com:80"); REQUIRE(url_h); REQUIRE(*url_h); // get with default opts auto ret_url = url_h.value()->GetURL(); REQUIRE(ret_url); CHECK(*ret_url == "http://example.com:80/"); // get with no default port ret_url = url_h.value()->GetURL(false, /* use_default_port */ false, /* use_default_scheme */ true /* use_no_default_port */ ); REQUIRE(ret_url); CHECK(*ret_url == "http://example.com/"); } SECTION("Get scheme of URL") { auto url_h = CurlURLHandle::Create("http://example.com:80"); REQUIRE(url_h); REQUIRE(*url_h); auto ret_url = url_h.value()->GetScheme(); REQUIRE(ret_url); CHECK(*ret_url == "http"); } SECTION("Duplicate URL") { auto url_h = CurlURLHandle::Create("http://example.com"); REQUIRE(url_h); REQUIRE(*url_h); auto url_h_dup = url_h.value()->Duplicate(); REQUIRE(url_h_dup); CHECK(url_h.value()->GetURL() == url_h_dup->GetURL()); } SECTION("Parse URL with permissive arguments") { // guess scheme from hostname auto url_h_guess_scheme = CurlURLHandle::CreatePermissive("ftp.example.com", true); CHECK(url_h_guess_scheme); CHECK(*url_h_guess_scheme); // check what was stored: path as-is stops after first slash auto ret_url_guess_scheme = url_h_guess_scheme.value()->GetURL(); REQUIRE(ret_url_guess_scheme); CHECK(*ret_url_guess_scheme == "ftp://ftp.example.com/"); // non-supported scheme, no authority, path not normalized auto url_h_nonstandard_scheme = CurlURLHandle::CreatePermissive( "socks5:///foo/../bar#boo", false, false, true, true, true); CHECK(url_h_nonstandard_scheme); CHECK(*url_h_nonstandard_scheme); // check what was stored: path as-is but without preceding slash auto ret_url_nonstandard_scheme = url_h_nonstandard_scheme.value()->GetURL(); REQUIRE(ret_url_nonstandard_scheme); CHECK(*ret_url_nonstandard_scheme == "socks5://foo/../bar#boo"); // bare IP treated with http scheme as default works (proxy-style) auto url_h_bare_ip = CurlURLHandle::CreatePermissive("192.0.2.1", true, false, true); CHECK(url_h_bare_ip); CHECK(*url_h_bare_ip); // check what was stored: defaults to http auto ret_url_bare_ip = url_h_bare_ip.value()->GetURL(); REQUIRE(ret_url_bare_ip); CHECK(*ret_url_bare_ip == "http://192.0.2.1/"); // default scheme, no authority, path normalized auto url_h_root_path = CurlURLHandle::CreatePermissive( "/", false, true, false, true, true); CHECK(url_h_root_path); CHECK(*url_h_root_path); // check what was stored: scheme http, path single slash auto ret_url_root_path = url_h_root_path.value()->GetURL(); REQUIRE(ret_url_root_path); CHECK(*ret_url_root_path == "https:///"); // empty url should fail (with even the most permissive options) auto url_h_empty = CurlURLHandle::CreatePermissive("", false, true, false, true, true); CHECK(url_h_empty); CHECK_FALSE(*url_h_empty); } SECTION("Parse config key") { auto key_h = CurlURLHandle::ParseConfigKey( "http://user@*.com/foo/bar?query#fragment"); CHECK(key_h); REQUIRE(*key_h); // check exact fields CHECK(key_h.value()->scheme); CHECK(key_h.value()->scheme.value() == "http"); CHECK(key_h.value()->user); CHECK(key_h.value()->user.value() == "user"); CHECK(key_h.value()->host); CHECK(key_h.value()->host.value() == "*.com"); CHECK(key_h.value()->port); CHECK(key_h.value()->port.value() == "80"); // default http port CHECK(key_h.value()->path.string() == "/foo/bar?query#fragment/"); } } TEST_CASE("Curl URL match config key", "[curl_url_match_config_key]") { auto url_h = CurlURLHandle::Create("http://user@example.com/foo/bar?query#fragment"); REQUIRE(url_h); REQUIRE(*url_h); SECTION("Match exactly") { auto match_exactly = url_h.value()->MatchConfigKey( "http://user@example.com/foo/bar?query#fragment"); REQUIRE(match_exactly); CHECK(match_exactly->matched); CHECK(match_exactly->host_len == 11); CHECK(match_exactly->path_len == 24); CHECK(match_exactly->user_matched); } SECTION("Match without user") { auto match_wo_user = url_h.value()->MatchConfigKey( "http://example.com/foo/bar?query#fragment"); REQUIRE(match_wo_user); CHECK(match_wo_user->matched); CHECK(match_wo_user->host_len == 11); CHECK(match_wo_user->path_len == 24); CHECK(not match_wo_user->user_matched); } SECTION("Match with default port") { auto match_default_port = url_h.value()->MatchConfigKey( "http://user@example.com:80/foo/bar?query#fragment"); REQUIRE(match_default_port); CHECK(match_default_port->matched); CHECK(match_default_port->host_len == 11); CHECK(match_default_port->path_len == 24); CHECK(match_default_port->user_matched); } SECTION("Match with path prefix") { auto match_path_prefix = url_h.value()->MatchConfigKey("http://user@example.com/foo"); REQUIRE(match_path_prefix); CHECK(match_path_prefix->matched); CHECK(match_path_prefix->host_len == 11); CHECK(match_path_prefix->path_len == 5); CHECK(match_path_prefix->user_matched); } SECTION("Match with path normalization") { auto match_path_normal = url_h.value()->MatchConfigKey( "http://user@example.com/./foo/boo/.."); REQUIRE(match_path_normal); CHECK(match_path_normal->matched); CHECK(match_path_normal->host_len == 11); CHECK(match_path_normal->path_len == 5); CHECK(match_path_normal->user_matched); } SECTION("Match with wildcarded host") { auto match_wildcard_host = url_h.value()->MatchConfigKey( "http://user@*.com/foo/bar?query#fragment"); REQUIRE(match_wildcard_host); CHECK(match_wildcard_host->matched); CHECK(match_wildcard_host->host_len == 5); CHECK(match_wildcard_host->path_len == 24); CHECK(match_wildcard_host->user_matched); } SECTION("Match with multiple wildcarded host") { auto match_wildcard_host = url_h.value()->MatchConfigKey( "http://user@*.*/foo/bar?query#fragment"); REQUIRE(match_wildcard_host); CHECK(match_wildcard_host->matched); CHECK(match_wildcard_host->host_len == 3); CHECK(match_wildcard_host->path_len == 24); CHECK(match_wildcard_host->user_matched); } SECTION("Match fail with unparsable key") { auto match_fail_parse = url_h.value()->MatchConfigKey("192.0.2.1"); REQUIRE(match_fail_parse); CHECK(not match_fail_parse->matched); CHECK(match_fail_parse->host_len == 0); CHECK(match_fail_parse->path_len == 0); CHECK(not match_fail_parse->user_matched); } SECTION("Match fail with wrong host") { auto match_fail_host = url_h.value()->MatchConfigKey( "http://user@example.org/foo/bar?query#fragment"); REQUIRE(match_fail_host); CHECK(not match_fail_host->matched); CHECK(match_fail_host->host_len == 0); CHECK(match_fail_host->path_len == 0); CHECK(not match_fail_host->user_matched); } SECTION("Match fail with wrong port") { auto match_fail_port = url_h.value()->MatchConfigKey( "http://user@example.com:1234/foo/bar?query#fragment"); REQUIRE(match_fail_port); CHECK(not match_fail_port->matched); CHECK(match_fail_port->host_len == 0); CHECK(match_fail_port->path_len == 0); CHECK(not match_fail_port->user_matched); } SECTION("Match fail with wrong path") { auto match_fail_path = url_h.value()->MatchConfigKey("http://user@example.com/foo/bar"); REQUIRE(match_fail_path); CHECK(not match_fail_path->matched); CHECK(match_fail_path->host_len == 0); CHECK(match_fail_path->path_len == 0); CHECK(not match_fail_path->user_matched); } } TEST_CASE("Curl URL match no_proxy patterns", "[curl_url_match_no-proxy]") { auto url_h = CurlURLHandle::Create( "http://user@example.com:50000/foo/bar?query#fragment"); REQUIRE(url_h); REQUIRE(*url_h); SECTION("Match with wildcard") { auto match_wildcard = url_h.value()->NoproxyStringMatches("*"); REQUIRE(match_wildcard); CHECK(*match_wildcard); } SECTION("Match with host") { auto match_host = url_h.value()->NoproxyStringMatches("example.com"); REQUIRE(match_host); CHECK(*match_host); } SECTION("Match with domain only") { auto match_domain = url_h.value()->NoproxyStringMatches("com"); REQUIRE(match_domain); CHECK(*match_domain); } SECTION("Match with stripped leading dot") { auto match_host_leading_dot = url_h.value()->NoproxyStringMatches(".example.com"); REQUIRE(match_host_leading_dot); CHECK(*match_host_leading_dot); } SECTION("Match with port") { auto match_port = url_h.value()->NoproxyStringMatches("example.com:50000"); REQUIRE(match_port); CHECK(*match_port); } SECTION("Match from multiple patterns") { auto match_check_parse = url_h.value()->NoproxyStringMatches("fail, wrong *"); REQUIRE(match_check_parse); CHECK(*match_check_parse); } SECTION("Match fail with wrong patterns") { auto match_fail = url_h.value()->NoproxyStringMatches("fail, wrong :50000,example"); REQUIRE(match_fail); CHECK(not *match_fail); } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/utils/curl_usage.test.cpp000066400000000000000000000046701516554100600301210ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/other_tools/utils/curl_context.hpp" #include "src/other_tools/utils/curl_easy_handle.hpp" // The caller of this test needs to make sure the port is given as content of // the file "port.txt" in the directory where this test is run [[nodiscard]] auto getPort() noexcept -> std::string { // read file where port has to be given auto port = FileSystemManager::ReadFile(std::filesystem::path("port.txt")); REQUIRE(port); // strip any end terminator std::erase_if(*port, [](auto ch) { return (ch == '\n' or ch == '\r'); }); return *port; } TEST_CASE("Curl context", "[curl_context]") { CurlContext curl_context{}; } TEST_CASE("Curl easy handle", "[curl_easy_handle]") { auto const serve_url = std::string("http://127.0.0.1:") + getPort() + std::string("/test_file.txt"); auto const target_dir = std::filesystem::path(std::getenv("TEST_TMPDIR")) / "target_dir"; // make target dir CHECK(FileSystemManager::CreateDirectory(target_dir)); // create handle auto curl_handle = CurlEasyHandle::Create(); REQUIRE(curl_handle); SECTION("Curl download to file") { // download test file from local HTTP server into new location auto file_path = target_dir / "test_file.txt"; REQUIRE(curl_handle->DownloadToFile(serve_url, file_path) == 0); REQUIRE(FileSystemManager::IsFile(file_path)); } SECTION("Curl download to string") { // download test file from local HTTP server into string auto content = curl_handle->DownloadToString(serve_url); REQUIRE(content); REQUIRE(*content == "test\n"); } } just-buildsystem-justbuild-b1fb5fa/test/other_tools/utils/curl_usage_test.sh000066400000000000000000000032421516554100600300240ustar00rootroot00000000000000#!/bin/sh # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu # cleanup of http.server; pass server_pid as arg server_cleanup() { echo "Shut down HTTP server" # send SIGTERM kill ${1} & res=$! wait ${res} echo "done" } readonly ROOT=`pwd` readonly SERVER_ROOT="${TEST_TMPDIR}/server-root" echo "Create test file" mkdir -p "${SERVER_ROOT}" cd "${SERVER_ROOT}" cat > test_file.txt < // ssize_t #else #error "Non-unix is not supported yet" #endif #include #include #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "catch2/generators/catch_generators_all.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/utils/archive/archive_ops.hpp" extern "C" { #include #include } namespace { using file_t = std::pair; using filetree_t = std::unordered_map; constexpr std::size_t kBlockSize = 10240; constexpr int kFilePerm = 0644; constexpr int kDirectoryPerm = 0755; auto const kExpected = filetree_t{{"foo", {"foo", AE_IFREG}}, {"bar/", {"", AE_IFDIR}}, {"bar/baz", {"baz", AE_IFREG}}}; struct ArchiveTestInfo { std::string test_name; ArchiveType type; std::string test_dir; std::string filename; std::vector tools; std::string cmd; }; std::vector const kTestScenarios = { {.test_name = "tar", .type = ArchiveType::Tar, .test_dir = "test_tar", .filename = "test.tar", .tools = {"tar"}, .cmd = "/usr/bin/tar xf"}, {.test_name = "tar.gz", .type = ArchiveType::TarGz, .test_dir = "test_tar_gz", .filename = "test.tar.gz", .tools = {"tar", "gzip"}, .cmd = "/usr/bin/tar xzf"}, {.test_name = "tar.bz2", .type = ArchiveType::TarBz2, .test_dir = "test_tar_bz2", .filename = "test.tar.bz2", .tools = {"tar", "bzip2"}, .cmd = "/usr/bin/tar xjf"}, {.test_name = "tar.xz", .type = ArchiveType::TarXz, .test_dir = "test_tar_xz", .filename = "test.tar.xz", .tools = {"tar", "xz"}, .cmd = "/usr/bin/tar xJf"}, {.test_name = "tar.lz", .type = ArchiveType::TarLz, .test_dir = "test_tar_lz", .filename = "test.tar.lz", .tools = {"tar", "lzip"}, .cmd = "/usr/bin/tar --lzip -x -f"}, {.test_name = "tar.lzma", .type = ArchiveType::TarLzma, .test_dir = "test_tar_lzma", .filename = "test.tar.lzma", .tools = {"tar", "lzma"}, .cmd = "/usr/bin/tar --lzma -x -f"}, {.test_name = "zip", .type = ArchiveType::Zip, .test_dir = "test_zip", .filename = "test.zip", .tools = {"unzip"}, .cmd = "/usr/bin/unzip"}, {.test_name = "7zip", .type = ArchiveType::_7Zip, .test_dir = "test_7zip", .filename = "test.7z", .tools = {"7z"}, // 7z comes with its own lzma-type compression .cmd = "/usr/bin/7z x"}}; [[nodiscard]] auto read_archive(archive* a, std::string const& path) -> filetree_t { filetree_t result{}; REQUIRE(archive_read_open_filename(a, path.c_str(), kBlockSize) == ARCHIVE_OK); archive_entry* entry{}; while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { auto size = archive_entry_size(entry); auto buf = std::string(static_cast(size), '\0'); REQUIRE(archive_read_data(a, buf.data(), buf.size()) == static_cast(buf.size())); result.emplace(archive_entry_pathname(entry), file_t{buf, archive_entry_filetype(entry)}); } REQUIRE(archive_read_close(a) == ARCHIVE_OK); return result; } void write_archive(archive* a, std::string const& path, filetree_t const& files) { REQUIRE(archive_write_open_filename(a, path.c_str()) == ARCHIVE_OK); archive_entry* entry = archive_entry_new(); for (auto const& [path, file] : files) { auto const& [content, type] = file; archive_entry_set_pathname(entry, path.c_str()); archive_entry_set_filetype(entry, type); if (type == AE_IFREG) { auto buf = std::filesystem::path{path}.filename().string(); archive_entry_set_perm(entry, kFilePerm); archive_entry_set_size(entry, static_cast(buf.size())); REQUIRE(archive_write_header(a, entry) == ARCHIVE_OK); REQUIRE(archive_write_data(a, buf.data(), buf.size()) == static_cast(buf.size())); } else { archive_entry_set_perm(entry, kDirectoryPerm); archive_entry_set_size(entry, 0); REQUIRE(archive_write_header(a, entry) == ARCHIVE_OK); } entry = archive_entry_clear(entry); } archive_entry_free(entry); REQUIRE(archive_write_close(a) == ARCHIVE_OK); } void extract_archive(std::string const& path) { auto* a = archive_read_new(); REQUIRE(a != nullptr); REQUIRE(archive_read_support_format_tar(a) == ARCHIVE_OK); REQUIRE(archive_read_support_format_zip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_format_7zip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_gzip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_bzip2(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_lzip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_lzma(a) == ARCHIVE_OK); REQUIRE(archive_read_open_filename(a, path.c_str(), kBlockSize) == ARCHIVE_OK); auto* out = archive_write_disk_new(); REQUIRE(out != nullptr); archive_entry* entry{}; int r{}; while ((r = archive_read_next_header(a, &entry)) == ARCHIVE_OK) { REQUIRE(archive_write_header(out, entry) == ARCHIVE_OK); if (archive_entry_size(entry) > 0) { void const* buf{}; std::size_t size{}; std::int64_t offset{}; int r2{}; while ((r2 = archive_read_data_block(a, &buf, &size, &offset)) == ARCHIVE_OK) { REQUIRE(archive_write_data_block(out, buf, size, offset) == ARCHIVE_OK); } REQUIRE(r2 == ARCHIVE_EOF); REQUIRE(archive_write_finish_entry(out) == ARCHIVE_OK); } } REQUIRE(r == ARCHIVE_EOF); REQUIRE(archive_read_close(a) == ARCHIVE_OK); REQUIRE(archive_read_free(a) == ARCHIVE_OK); REQUIRE(archive_write_close(out) == ARCHIVE_OK); REQUIRE(archive_write_free(out) == ARCHIVE_OK); } void compare_extracted( std::filesystem::path const& extract_dir = ".") noexcept { for (auto const& [path, file] : kExpected) { auto const& [content, type] = file; switch (type) { case AE_IFREG: { REQUIRE(FileSystemManager::IsFile(extract_dir / path)); auto data = FileSystemManager::ReadFile(extract_dir / path); REQUIRE(data); CHECK(*data == content); } break; case AE_IFDIR: CHECK(FileSystemManager::IsDirectory(extract_dir / path)); break; default: CHECK(false); } } } void create_files(std::filesystem::path const& destDir = ".") noexcept { for (auto const& [path, file] : kExpected) { auto const& [content, type] = file; switch (type) { case AE_IFREG: { CHECK(FileSystemManager::WriteFile(content, destDir / path)); } break; case AE_IFDIR: CHECK(FileSystemManager::CreateDirectory(destDir / path)); break; default: CHECK(false); } } } void enable_write_format_and_filter(archive* aw, ArchiveType type) { switch (type) { case ArchiveType::Zip: { REQUIRE(archive_write_set_format_zip(aw) == ARCHIVE_OK); } break; case ArchiveType::_7Zip: { REQUIRE(archive_write_set_format_7zip(aw) == ARCHIVE_OK); } break; case ArchiveType::Tar: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); } break; case ArchiveType::TarGz: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_gzip(aw) == ARCHIVE_OK); } break; case ArchiveType::TarBz2: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_bzip2(aw) == ARCHIVE_OK); } break; case ArchiveType::TarXz: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_xz(aw) == ARCHIVE_OK); } break; case ArchiveType::TarLz: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_lzip(aw) == ARCHIVE_OK); } break; case ArchiveType::TarLzma: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_lzma(aw) == ARCHIVE_OK); } break; case ArchiveType::ZipAuto: case ArchiveType::TarAuto: return; // unused } } void enable_read_format_and_filter(archive* ar, ArchiveType type) { switch (type) { case ArchiveType::Zip: { REQUIRE(archive_read_support_format_zip(ar) == ARCHIVE_OK); } break; case ArchiveType::_7Zip: { REQUIRE(archive_read_support_format_7zip(ar) == ARCHIVE_OK); } break; case ArchiveType::Tar: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); } break; case ArchiveType::TarGz: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_gzip(ar) == ARCHIVE_OK); } break; case ArchiveType::TarBz2: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); } break; case ArchiveType::TarXz: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(ar) == ARCHIVE_OK); } break; case ArchiveType::TarLz: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_lzip(ar) == ARCHIVE_OK); } break; case ArchiveType::TarLzma: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_lzma(ar) == ARCHIVE_OK); } break; case ArchiveType::ZipAuto: case ArchiveType::TarAuto: return; // unused } } } // namespace TEST_CASE("Archive read context", "[archive_context]") { auto* a = archive_read_new(); REQUIRE(a != nullptr); CHECK(archive_read_free(a) == ARCHIVE_OK); } TEST_CASE("Archive write context", "[archive_context]") { auto* a = archive_write_new(); REQUIRE(a != nullptr); CHECK(archive_write_free(a) == ARCHIVE_OK); } TEST_CASE("Archive write disk context", "[archive_context]") { auto* a = archive_write_disk_new(); REQUIRE(a != nullptr); CHECK(archive_read_free(a) == ARCHIVE_OK); } TEST_CASE("Read-write archives", "[archive_read_write]") { // get the scenario auto test_index = GENERATE( Catch::Generators::range(0, kTestScenarios.size())); auto const& scenario = kTestScenarios[test_index]; // perform the test REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir)); REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); auto anchor = FileSystemManager::ChangeDirectory(scenario.test_dir); SECTION(std::string("Write ") + scenario.test_name) { auto* out = archive_write_new(); REQUIRE(out != nullptr); enable_write_format_and_filter(out, scenario.type); write_archive(out, scenario.filename, kExpected); REQUIRE(archive_write_free(out) == ARCHIVE_OK); SECTION(std::string("Read ") + scenario.test_name) { auto* in = archive_read_new(); REQUIRE(in != nullptr); enable_read_format_and_filter(in, scenario.type); CHECK(read_archive(in, scenario.filename) == kExpected); REQUIRE(archive_read_free(in) == ARCHIVE_OK); } SECTION(std::string("Extract ") + scenario.test_name + " to disk") { extract_archive(scenario.filename); compare_extracted(); } bool tools_exist{true}; for (auto const& tool : scenario.tools) { tools_exist &= FileSystemManager::IsExecutable( std::string("/usr/bin/") + tool); } if (tools_exist) { std::string path{"/usr/bin"}; if (auto* env_path = std::getenv("PATH")) { path = std::string{env_path} + ":" + path; } SECTION("Extract via system tools") { REQUIRE(system(("export PATH=" + path + " && " + scenario.cmd + " " + scenario.filename) .c_str()) == 0); compare_extracted(); } } } } TEST_CASE("ArchiveOps", "[archive_ops]") { // get the scenario auto test_index = GENERATE( Catch::Generators::range(0, kTestScenarios.size())); auto const& scenario = kTestScenarios[test_index]; // perform the test std::optional res{std::nullopt}; SECTION(std::string("Write ") + scenario.test_name) { REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir)); REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); create_files(scenario.test_dir); res = ArchiveOps::CreateArchive( scenario.type, scenario.filename, scenario.test_dir, "."); if (res != std::nullopt) { FAIL(*res); } SECTION(std::string("Extract ") + scenario.test_name + " to disk") { REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir)); REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); res = ArchiveOps::ExtractArchive( scenario.type, scenario.filename, "."); if (res != std::nullopt) { FAIL(*res); } compare_extracted(scenario.test_dir); } bool tools_exist{true}; for (auto const& tool : scenario.tools) { tools_exist &= FileSystemManager::IsExecutable( std::string("/usr/bin/") + tool); } if (tools_exist) { std::string path{"/usr/bin"}; if (auto* env_path = std::getenv("PATH")) { path = std::string{env_path} + ":" + path; } SECTION("Extract via system tools") { REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir)); REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); REQUIRE(system(("export PATH=" + path + " && " + scenario.cmd + " " + scenario.filename) .c_str()) == 0); compare_extracted(scenario.test_dir); } } } } just-buildsystem-justbuild-b1fb5fa/test/utils/connection_documenting_server.py000077500000000000000000000022431516554100600304350ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import socketserver log_file = "access.log" class MyTCPHandler(socketserver.BaseRequestHandler): def handle(self): with open(log_file, "a") as f: f.write("Connected from {}\n".format(self.client_address,)) self.request.sendall(b"Wrong, but thanks for playing") log_file = sys.argv[2] with socketserver.TCPServer(("127.0.0.1", 0), MyTCPHandler) as server: socket_info = server.socket.getsockname() with open(sys.argv[1], "w") as f: f.write("%d" % (socket_info[1],)) server.serve_forever() just-buildsystem-justbuild-b1fb5fa/test/utils/container_matchers.hpp000066400000000000000000000160721516554100600263250ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_CONTAINER_MATCHERS_HPP #define INCLUDED_SRC_TEST_UTILS_CONTAINER_MATCHERS_HPP #include #include #include #include #include #include #include #include #include "catch2/matchers/catch_matchers_all.hpp" /// \brief Matcher to check if the sets of elements present in two different /// containers are the same template class UniqueElementsUnorderedMatcher : public Catch::Matchers::MatcherBase { RightContainer rhs_; public: // Note that in the case of an associative container with type C, // C::value_type == std::pair. // That would be a problem as we will be using // std::unordered_set So don't use this class in the case you // want to compare two (multi)maps (ordered/unordered) using value_type = typename LeftContainer::value_type; using T = value_type; static_assert( std::is_constructible_v, "Value type of container in the left hand side must be constructible " "from that of the right hand side."); explicit UniqueElementsUnorderedMatcher(RightContainer const& rc) : rhs_(rc) {} UniqueElementsUnorderedMatcher() = delete; // Method that produces the result to be evaluated [[nodiscard]] auto match(LeftContainer const& lc) const -> bool override { return IsEqualToRHS( std::unordered_set(std::begin(lc), std::end(lc))); } [[nodiscard]] auto describe() const -> std::string override { std::ostringstream ss; ss << "\nhas the same unique elements as\n{"; auto elem_it = std::begin(rhs_); if (elem_it != std::end(rhs_)) { ss << *elem_it; ++elem_it; for (; elem_it != std::end(rhs_); ++elem_it) { ss << ", " << *elem_it; } } ss << "}."; return ss.str(); } private: [[nodiscard]] auto IsEqualToRHS(std::unordered_set const& lhs) const -> bool { std::unordered_set rhs(std::begin(rhs_), std::end(rhs_)); for (auto const& elem : lhs) { auto elem_it_rhs = rhs.find(elem); if (elem_it_rhs == std::end(rhs)) { return false; } rhs.erase(elem_it_rhs); } return rhs.empty(); } }; template inline auto HasSameUniqueElementsAs(RightContainer const& rc) -> UniqueElementsUnorderedMatcher { return UniqueElementsUnorderedMatcher(rc); } template inline auto HasSameUniqueElementsAs(std::initializer_list const& rc) -> UniqueElementsUnorderedMatcher> { return UniqueElementsUnorderedMatcher>(rc); } /// \brief Matcher to compare the contents of two containers up to permutation template class ContainerUnorderedMatcher : public Catch::Matchers::MatcherBase { public: using value_type = typename LeftContainer::value_type; using T = value_type; explicit ContainerUnorderedMatcher(std::vector const& rc) : rhs_(rc) {} ContainerUnorderedMatcher() = delete; // Method that produces the result to be evaluated [[nodiscard]] auto match(LeftContainer const& lc) const -> bool override { return IsEqualToRHS(std::vector(std::begin(lc), std::end(lc))); } [[nodiscard]] auto describe() const -> std::string override { std::ostringstream ss; ss << "\nhas the same elements as\n{"; auto elem_it = std::begin(rhs_); if (elem_it != std::end(rhs_)) { ss << *elem_it; ++elem_it; for (; elem_it != std::end(rhs_); ++elem_it) { ss << ", " << *elem_it; } } ss << "}."; return ss.str(); } private: std::vector rhs_; /// \brief Compare containers by checking they have the same elements /// (repetitions included). This implementation is not optimal, but it /// doesn't require that the type T = LeftContainer::value_type has /// known-to-STL hashing function or partial order (<) [[nodiscard]] auto IsEqualToRHS(std::vector const& lhs) const -> bool { if (std::size(lhs) != std::size(rhs_)) { return false; } // Get iterators to the rhs vector, we will remove iterators of elements // found from this vector in order to account for repetitions std::vector::const_iterator> iterators_to_check( rhs_.size()); std::iota(std::begin(iterators_to_check), std::end(iterators_to_check), std::begin(rhs_)); // Instead of removing elements from the vector, as this would mean // moving O(n) of them, we swap them to the back of the vector and keep // track of what's the last element that has to be checked. // This is similar to std::remove, but we are only interested in doing // it for one element at a time. auto last_to_check = std::end(iterators_to_check); auto check_exists_and_remove = [&iterators_to_check, &last_to_check](T const& elem) { auto it_to_elem = std::find_if(std::begin(iterators_to_check), last_to_check, [&elem](auto iter) { return *iter == elem; }); if (it_to_elem == last_to_check) { return false; } --last_to_check; std::iter_swap(it_to_elem, last_to_check); return true; }; return std::all_of(lhs.begin(), lhs.end(), [&check_exists_and_remove](auto const& element) { return check_exists_and_remove(element); }); } }; template inline auto HasSameElementsAs( std::vector const& rc) -> ContainerUnorderedMatcher { return ContainerUnorderedMatcher(rc); } #endif // INCLUDED_SRC_TEST_UTILS_CONTAINER_MATCHERS_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/000077500000000000000000000000001516554100600225205ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/TARGETS000066400000000000000000000046561516554100600235670ustar00rootroot00000000000000{ "path": { "type": ["@", "rules", "CC/test", "test"] , "name": ["path"] , "srcs": ["path.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/utils/cpp", "path"] , ["", "catch-main"] ] , "stage": ["test", "utils", "cpp"] } , "path_rebase": { "type": ["@", "rules", "CC/test", "test"] , "name": ["path_rebase"] , "srcs": ["path_rebase.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/utils/cpp", "path_rebase"] , ["", "catch-main"] ] , "stage": ["test", "utils", "cpp"] } , "file_locking": { "type": ["@", "rules", "CC/test", "test"] , "name": ["file_locking"] , "srcs": ["file_locking.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/utils/cpp", "atomic"] , ["@", "src", "src/utils/cpp", "file_locking"] , ["", "catch-main"] ] , "stage": ["test", "utils", "cpp"] , "private-ldflags": ["-pthread"] } , "prefix": { "type": ["@", "rules", "CC/test", "test"] , "name": ["prefix"] , "srcs": ["prefix.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/utils/cpp", "prefix"] , ["", "catch-main"] ] , "stage": ["test", "utils", "cpp"] } , "tmp_dir": { "type": ["@", "rules", "CC/test", "test"] , "name": ["tmp_dir"] , "srcs": ["tmp_dir.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/file_system", "file_system_manager"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] ] , "stage": ["test", "utils", "cpp"] } , "incremental_reader": { "type": ["@", "rules", "CC/test", "test"] , "name": ["incremental_reader"] , "srcs": ["incremental_reader.test.cpp"] , "private-deps": [ ["@", "catch2", "", "catch2"] , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["@", "src", "src/utils/cpp", "incremental_reader"] , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "large_object_utils"] , ["utils", "test_storage_config"] ] , "stage": ["test", "utils", "cpp"] } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["cpp"] , "deps": [ "file_locking" , "incremental_reader" , "path" , "path_rebase" , "prefix" , "tmp_dir" ] } } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/file_locking.test.cpp000066400000000000000000000047611516554100600266370ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/file_locking.hpp" #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/utils/cpp/atomic.hpp" namespace { [[nodiscard]] auto GetTestDir() noexcept -> std::filesystem::path { auto* tmp_dir = std::getenv("TEST_TMPDIR"); if (tmp_dir != nullptr) { return tmp_dir; } return FileSystemManager::GetCurrentDirectory() / "test/other_tools"; } [[nodiscard]] auto GetLockDirPath(int id) noexcept -> std::filesystem::path { auto lock_file = std::to_string(id) + std::string{".lock"}; return GetTestDir() / lock_file; } } // namespace TEST_CASE("Multi-file locking", "[file_locking]") { // Test locking and unlocking. Each thread will have one lock. // setup threading constexpr auto kNumThreads = 50; // increasing it too much will fail constexpr auto kNumLocks = 5; atomic starting_signal{false}; std::vector threads{}; threads.reserve(kNumThreads); for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&starting_signal](int tid) { starting_signal.wait(false); // cases based on id auto flockpath = GetLockDirPath(tid % kNumLocks); // Get lock auto lock = LockFile::Acquire(flockpath, /*is_shared=*/false); REQUIRE(lock); // Do some "work" std::this_thread::sleep_for(std::chrono::milliseconds(1)); // lock released automatically when out of scope }, id); } starting_signal = true; starting_signal.notify_all(); // wait for threads to finish for (auto& thread : threads) { thread.join(); } } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/incremental_reader.test.cpp000066400000000000000000000127061516554100600300330ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/incremental_reader.hpp" #include #include #include #include #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/large_objects/large_object_utils.hpp" [[nodiscard]] static auto ReadFile(std::filesystem::path const& path) noexcept -> std::optional { try { std::size_t const file_size = std::filesystem::file_size(path); std::string result(file_size, '\0'); std::ifstream istream(path.string(), std::ios_base::binary); if (not istream.good()) { return std::nullopt; } istream.read(result.data(), static_cast(file_size)); return result; } catch (...) { return std::nullopt; } } TEST_CASE("IncrementalReader", "[incremental_reader]") { static constexpr auto kFileSize = static_cast(5 * 1024 * 1024); static constexpr std::size_t kChunkWithRemainder = 107; static_assert(kFileSize % kChunkWithRemainder != 0); static constexpr std::size_t kChunkWithoutRemainder = 128; static_assert(kFileSize % kChunkWithoutRemainder == 0); auto const config = TestStorageConfig::Create(); auto temp_dir = config.Get().CreateTypedTmpDir("incremental_reader"); REQUIRE(temp_dir); auto const file_path = temp_dir->GetPath() / "file"; REQUIRE(LargeObjectUtils::GenerateFile(file_path, kFileSize)); auto const file_content = ::ReadFile(file_path); REQUIRE(file_content.has_value()); SECTION("File") { auto reader = IncrementalReader::FromFile(kChunkWithRemainder, file_path); REQUIRE(reader.has_value()); std::string result; result.reserve(reader->GetContentSize()); for (auto chunk : *reader) { REQUIRE(chunk.has_value()); result.append(*chunk); } REQUIRE(result.size() == file_content->size()); REQUIRE(result == *file_content); reader = IncrementalReader::FromFile(kChunkWithoutRemainder, file_path); REQUIRE(reader.has_value()); result.clear(); for (auto chunk : *reader) { REQUIRE(chunk.has_value()); result.append(*chunk); } REQUIRE(result.size() == file_content->size()); REQUIRE(result == *file_content); } SECTION("Memory") { auto reader = IncrementalReader::FromMemory(kChunkWithRemainder, &*file_content); REQUIRE(reader.has_value()); std::string result; result.reserve(reader->GetContentSize()); for (auto chunk : *reader) { REQUIRE(chunk.has_value()); result.append(*chunk); } REQUIRE(result.size() == file_content->size()); REQUIRE(result == *file_content); reader = IncrementalReader::FromMemory(kChunkWithoutRemainder, &*file_content); REQUIRE(reader.has_value()); result.clear(); for (auto chunk : *reader) { REQUIRE(chunk.has_value()); result.append(*chunk); } REQUIRE(result.size() == file_content->size()); REQUIRE(result == *file_content); } } TEST_CASE("IncrementalReader - Empty", "[incremental_reader]") { static constexpr std::size_t kChunkSize = 128; SECTION("File") { auto const config = TestStorageConfig::Create(); auto temp_dir = config.Get().CreateTypedTmpDir("incremental_reader"); REQUIRE(temp_dir); std::filesystem::path const empty_file = temp_dir->GetPath() / "file"; { std::ofstream stream(empty_file); stream << ""; } auto const reader = IncrementalReader::FromFile(kChunkSize, empty_file); REQUIRE(reader.has_value()); std::optional result; for (auto chunk : *reader) { REQUIRE(chunk.has_value()); if (not result.has_value()) { result = std::string{}; } result->append(*chunk); } REQUIRE(result.has_value()); REQUIRE(result->empty()); } SECTION("Memory") { std::string const empty; auto const reader = IncrementalReader::FromMemory(kChunkSize, &empty); REQUIRE(reader.has_value()); std::optional result; for (auto chunk : *reader) { REQUIRE(chunk.has_value()); if (not result.has_value()) { result = std::string{}; } result->append(*chunk); } REQUIRE(result.has_value()); REQUIRE(result->empty()); } } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/path.test.cpp000066400000000000000000000043641516554100600251450ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/path.hpp" #include #include #include "catch2/catch_test_macros.hpp" TEST_CASE("normalization", "[path]") { CHECK(ToNormalPath(std::filesystem::path{""}) == ToNormalPath(std::filesystem::path{"."})); CHECK(ToNormalPath(std::filesystem::path{""}).string() == "."); CHECK(ToNormalPath(std::filesystem::path{"."}).string() == "."); CHECK(ToNormalPath(std::filesystem::path{"foo/bar/.."}).string() == "foo"); CHECK(ToNormalPath(std::filesystem::path{"foo/bar/../"}).string() == "foo"); CHECK(ToNormalPath(std::filesystem::path{"foo/bar/../baz"}).string() == "foo/baz"); CHECK(ToNormalPath(std::filesystem::path{"./foo/bar"}).string() == "foo/bar"); CHECK(ToNormalPath(std::filesystem::path{"foo/.."}).string() == "."); CHECK(ToNormalPath(std::filesystem::path{"./foo/.."}).string() == "."); } TEST_CASE("non-upwards condition", "[path]") { CHECK_FALSE(PathIsNonUpwards("/foo")); // absolute path CHECK(PathIsNonUpwards("foo")); // relative non-upwards CHECK_FALSE(PathIsNonUpwards("../foo")); // relative not non-upwards CHECK_FALSE(PathIsNonUpwards( "foo/../bar/../../foo")); // relative with non-upwards indirection } TEST_CASE("confined upwards condition", "[path]") { CHECK_FALSE(PathIsConfined("/foo", "dummy")); // absolute path CHECK(PathIsConfined("foo", "dummy")); // relative non-upwards CHECK(PathIsConfined("../foo", "dummy/bar")); // upwards, but confined CHECK_FALSE(PathIsConfined("foo/../bar/../../../foo", "dummy")); // upwards, not confined } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/path_rebase.test.cpp000066400000000000000000000037761516554100600264740ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/path_rebase.hpp" #include #include #include "catch2/catch_test_macros.hpp" TEST_CASE("rebase", "[path_rebase]") { CHECK(RebasePathStringRelativeTo("work", "work/foo/bar") == "foo/bar"); CHECK(RebasePathStringRelativeTo("work", "work/foo") == "foo"); CHECK(RebasePathStringRelativeTo("work", "work") == "."); CHECK(RebasePathStringRelativeTo("work", "other/foo.txt") == "../other/foo.txt"); CHECK(RebasePathStringRelativeTo("work/foo", "foo.txt") == "../../foo.txt"); CHECK(RebasePathStringRelativeTo("work/foo", "work/foo/bar") == "bar"); CHECK(RebasePathStringRelativeTo("work/foo", "work/foo") == "."); CHECK(RebasePathStringRelativeTo("work/foo", "work") == ".."); } TEST_CASE("no change", "[path_rebase]") { CHECK(RebasePathStringRelativeTo("", "work/foo/bar") == "work/foo/bar"); CHECK(RebasePathStringRelativeTo(".", "work/foo/bar") == "work/foo/bar"); CHECK(RebasePathStringRelativeTo("", ".") == "."); CHECK(RebasePathStringRelativeTo("", "") == "."); } TEST_CASE("vector rebse", "[path_rebase]") { std::vector input{ "work/foo.txt", "work/bar/baz.txt", "other/out.txt"}; auto output = RebasePathStringsRelativeTo("work", input); REQUIRE(output.size() == 3); CHECK(output[0] == "foo.txt"); CHECK(output[1] == "bar/baz.txt"); CHECK(output[2] == "../other/out.txt"); } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/prefix.test.cpp000066400000000000000000000017601516554100600255030ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/prefix.hpp" #include #include "catch2/catch_test_macros.hpp" TEST_CASE("lines", "[prefix]") { CHECK(PrefixLines("").empty()); CHECK(PrefixLines("foo\nbar\n") == " foo\n bar\n"); CHECK(PrefixLines("foo\nbar\n", ">") == ">foo\n>bar\n"); CHECK(PrefixLines("foo") == " foo\n"); CHECK(PrefixLines("foo\nbar") == " foo\n bar\n"); } just-buildsystem-justbuild-b1fb5fa/test/utils/cpp/tmp_dir.test.cpp000066400000000000000000000115131516554100600256410ustar00rootroot00000000000000// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/utils/cpp/tmp_dir.hpp" #include #include #include #include #include "catch2/catch_test_macros.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" TEST_CASE("tmp_dir", "[tmp_dir]") { char const* const env_tmpdir = std::getenv("TEST_TMPDIR"); REQUIRE(env_tmpdir != nullptr); auto const test_tempdir = std::filesystem::path{std::string{env_tmpdir}} / ".test_build_root"; REQUIRE(FileSystemManager::CreateDirectory(test_tempdir)); SECTION("simple") { // Create a temp directory auto temp_dir = TmpDir::Create(test_tempdir / "test_dir"); REQUIRE(temp_dir != nullptr); // Create one more temp directory at the same location to ensure the // template gets populated, and a new directory gets created auto temp_dir_2 = TmpDir::Create(test_tempdir / "test_dir"); REQUIRE(temp_dir_2 != nullptr); std::filesystem::path const temp_dir_path = temp_dir->GetPath(); REQUIRE(FileSystemManager::Exists(temp_dir_path)); { auto temp_dir_clone = temp_dir; temp_dir = nullptr; REQUIRE(FileSystemManager::Exists(temp_dir_path)); } REQUIRE(not FileSystemManager::Exists(temp_dir_path)); std::filesystem::path const temp_dir_path_2 = temp_dir_2->GetPath(); REQUIRE(FileSystemManager::Exists(temp_dir_path_2)); { auto temp_dir_clone = temp_dir_2; temp_dir_2 = nullptr; REQUIRE(FileSystemManager::Exists(temp_dir_path_2)); } REQUIRE(not FileSystemManager::Exists(temp_dir_path_2)); } SECTION("nested directories") { auto parent_dir = TmpDir::Create(test_tempdir / "test_dir"); REQUIRE(parent_dir != nullptr); std::filesystem::path const parent = parent_dir->GetPath(); auto child_dir_1 = TmpDir::CreateNestedDirectory(parent_dir); REQUIRE(child_dir_1); std::filesystem::path const child_1 = child_dir_1->GetPath(); auto child_dir_2 = TmpDir::CreateNestedDirectory(parent_dir); REQUIRE(child_dir_2); std::filesystem::path const child_2 = child_dir_2->GetPath(); REQUIRE(FileSystemManager::Exists(parent)); REQUIRE(FileSystemManager::Exists(child_1)); REQUIRE(FileSystemManager::Exists(child_2)); // Kill the parent directory. child_1 and child_2 still retain // references to the parent object, so all folders should remain alive: parent_dir = nullptr; REQUIRE(FileSystemManager::Exists(parent)); REQUIRE(FileSystemManager::Exists(child_1)); REQUIRE(FileSystemManager::Exists(child_2)); // Kill child_1. child_1 dies, but child_2 retains a reference to the // parent directory, so parent and child_2 must be alive: child_dir_1 = nullptr; REQUIRE(FileSystemManager::Exists(parent)); REQUIRE(not FileSystemManager::Exists(child_1)); REQUIRE(FileSystemManager::Exists(child_2)); // Kill child_2. All directories should be destroyed: child_dir_2 = nullptr; REQUIRE(not FileSystemManager::Exists(parent)); REQUIRE(not FileSystemManager::Exists(child_1)); REQUIRE(not FileSystemManager::Exists(child_2)); } SECTION("temp files") { auto parent_dir = TmpDir::Create(test_tempdir / "test_dir"); REQUIRE(parent_dir != nullptr); std::filesystem::path const parent = parent_dir->GetPath(); auto file = TmpDir::CreateFile(parent_dir); REQUIRE(file != nullptr); std::filesystem::path const file_path = file->GetPath(); REQUIRE(FileSystemManager::Exists(parent)); REQUIRE(FileSystemManager::Exists(file_path)); // Kill the parent directory. File still retains a reference to the // parent object, so parent should remain alive: parent_dir = nullptr; REQUIRE(FileSystemManager::Exists(parent)); REQUIRE(FileSystemManager::Exists(file_path)); // Kill the file. Both the parent directory and the file should be // deleted: file = nullptr; REQUIRE(not FileSystemManager::Exists(parent)); REQUIRE(not FileSystemManager::Exists(file_path)); } } just-buildsystem-justbuild-b1fb5fa/test/utils/executor/000077500000000000000000000000001516554100600235745ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/executor/test_api_bundle.hpp000066400000000000000000000023001516554100600274410ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_EXECUTOR_TEST_API_BUNDLE_HPP #define INCLUDED_SRC_TEST_UTILS_EXECUTOR_TEST_API_BUNDLE_HPP #include "gsl/gsl" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" /// \brief Creates an ApiBundle that contains a given IExecutionApi /// implementation. [[nodiscard]] static auto CreateTestApiBundle( gsl::not_null const& api) noexcept -> ApiBundle { return ApiBundle{.local = api, .remote = api}; } #endif // INCLUDED_SRC_TEST_UTILS_EXECUTOR_TEST_API_BUNDLE_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/hermeticity/000077500000000000000000000000001516554100600242645ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/hermeticity/test_hash_function_type.hpp000066400000000000000000000030361516554100600317270ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_HASH_FUNCTION_TYPE_HPP #define INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_HASH_FUNCTION_TYPE_HPP #include #include #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "test/utils/test_env.hpp" class TestHashType final { public: [[nodiscard]] static auto ReadFromEnvironment() noexcept -> HashFunction::Type { auto const compatible = ReadCompatibilityFromEnv(); if (not compatible) { Logger::Log(LogLevel::Error, "Failed to read COMPATIBLE from environment"); std::exit(EXIT_FAILURE); } return *compatible ? HashFunction::Type::PlainSHA256 : HashFunction::Type::GitSHA1; } }; #endif // INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_HASH_FUNCTION_TYPE_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/hermeticity/test_storage_config.hpp000066400000000000000000000066111516554100600310310ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_STORAGE_CONFIG_HPP #define INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_STORAGE_CONFIG_HPP #include //std::exit, std::getenv #include #include #include #include //std::move #include "gsl/gsl" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" #include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" class TestStorageConfig final { public: /// \brief Create a unique StorageConfig that has the build root in a new /// empty location. Uses TEST_TMPDIR environment variable to determine path /// to the location. /// To be used only for local tests, as it does not know about remote /// execution config. [[nodiscard]] static auto Create() noexcept -> TestStorageConfig { /** * Test must not assume the existence of a home directory, nor write * there. Hence we set the storage root to a fixed location under * TEST_TMPDIR which is set by the test launcher. */ char const* const env_tmpdir = std::getenv("TEST_TMPDIR"); if (env_tmpdir == nullptr) { Logger::Log(LogLevel::Debug, "missing TEST_TMPDIR env variable"); std::exit(EXIT_FAILURE); } auto const test_tempdir = std::filesystem::path{std::string{env_tmpdir}} / ".test_build_root"; auto temp_dir = TmpDir::Create(test_tempdir); if (temp_dir == nullptr) { Logger::Log(LogLevel::Error, "failed to create a test-local cache dir"); std::exit(EXIT_FAILURE); } StorageConfig::Builder builder; auto config = builder.SetBuildRoot(temp_dir->GetPath()) .SetHashType(TestHashType::ReadFromEnvironment()) .Build(); if (not config) { Logger::Log(LogLevel::Error, config.error()); std::exit(EXIT_FAILURE); } Logger::Log(LogLevel::Debug, "created test-local cache dir {}", temp_dir->GetPath().string()); return TestStorageConfig{std::move(temp_dir), *std::move(config)}; } [[nodiscard]] auto Get() const noexcept -> StorageConfig const& { return storage_config_; } private: gsl::not_null const tmp_dir_; StorageConfig const storage_config_; explicit TestStorageConfig(gsl::not_null const& tmp_dir, StorageConfig config) noexcept : tmp_dir_{tmp_dir}, storage_config_{std::move(config)} {} }; #endif // INCLUDED_SRC_TEST_UTILS_HERMETICITY_TEST_STORAGE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/large_objects/000077500000000000000000000000001516554100600245415ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/large_objects/large_object_utils.cpp000066400000000000000000000126101516554100600311050ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "test/utils/large_objects/large_object_utils.hpp" #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "src/buildtool/file_system/file_system_manager.hpp" namespace { class Randomizer final { public: Randomizer(std::uint64_t min, std::uint64_t max) noexcept : range_(std::random_device{}()), distribution_(min, max) {} [[nodiscard]] auto Get() noexcept -> std::uint64_t { return distribution_(range_); } private: std::mt19937_64 range_; std::uniform_int_distribution distribution_; }; /// \brief Create a number of chunks of the predefined size. /// \tparam UChunkLength Length of each chunk. /// \tparam USize Number of chunks. template class ChunkPool final { public: [[nodiscard]] static auto Instance() noexcept -> ChunkPool const& { static ChunkPool pool; return pool; } [[nodiscard]] auto operator[](std::size_t index) const noexcept -> std::string const& { return gsl::at(pool_, gsl::narrow(index)); } private: std::array pool_; explicit ChunkPool() noexcept { // Starts from 1 to exclude '\0' from randomization Randomizer randomizer{1, std::numeric_limits::max()}; for (std::size_t i = 0; i < pool_.size(); ++i) { auto& chunk = gsl::at(pool_, gsl::narrow(i)); chunk.resize(kChunkLength); for (std::size_t j = 0; j < kChunkLength; ++j) { chunk[j] = randomizer.Get(); } } } }; } // namespace auto LargeObjectUtils::GenerateFile(std::filesystem::path const& path, std::uintmax_t size, bool is_executable) noexcept -> bool { // Remove the file, if exists: if (not FileSystemManager::RemoveFile(path)) { return false; } static constexpr std::size_t kChunkLength = 128; static constexpr std::size_t kPoolSize = 64; using Pool = ChunkPool; // To create a random file, the initial chunk position and the shift are // randomized: Randomizer randomizer{std::numeric_limits::min(), std::numeric_limits::max()}; const std::size_t pool_index = randomizer.Get() % kPoolSize; const std::size_t pool_shift = randomizer.Get() % 10; const std::size_t step_count = size / kChunkLength + 1; try { std::ofstream stream(path); for (std::size_t i = 0; i < step_count and stream.good(); ++i) { const std::size_t index = (pool_index + i * pool_shift) % kPoolSize; if (i != step_count - 1) { stream << Pool::Instance()[index]; } else { auto count = std::min(size - kChunkLength * i, kChunkLength); stream << Pool::Instance()[index].substr(0, count); } } if (not stream.good()) { return false; } stream.close(); } catch (...) { return false; } if (is_executable) { using perms = std::filesystem::perms; perms p = perms::owner_exec | perms::group_exec | perms::others_exec; try { std::filesystem::permissions( path, p, std::filesystem::perm_options::add); } catch (...) { return false; } } return true; } auto LargeObjectUtils::GenerateDirectory(std::filesystem::path const& path, std::uintmax_t entries_count) noexcept -> bool { // Recreate the directory: if (not FileSystemManager::RemoveDirectory(path) or not FileSystemManager::CreateDirectory(path)) { return false; } Randomizer randomizer{std::numeric_limits::min(), std::numeric_limits::max()}; std::uintmax_t entries = 0; while (entries < entries_count) { // Randomize the number for a file: auto const random_number = randomizer.Get(); auto const file_name = std::string(kTreeEntryPrefix) + std::to_string(random_number); std::filesystem::path const file_path = path / file_name; // Check file uniqueness: if (FileSystemManager::IsFile(file_path)) { continue; } try { std::ofstream stream(file_path); stream << random_number; stream.close(); } catch (...) { return false; } ++entries; } return true; } just-buildsystem-justbuild-b1fb5fa/test/utils/large_objects/large_object_utils.hpp000066400000000000000000000047561516554100600311260ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP #define INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP #include #include #include /// \brief Provides an interface for randomizing large files and directories. class LargeObjectUtils { public: static constexpr std::string_view kTreeEntryPrefix = "additional-large-prefix-to-make-tree-entry-larger"; /// \brief Generate a file of the specified size in the specified location. /// If the file exists, it is overwritten. To reduce the number of /// randomizations, a pool of pre-generated chunks is used. /// \param path Output path. /// \param size Size of the resulting file in bytes. /// \param is_executable Set executable permissions /// \return True if the file is generated properly. [[nodiscard]] static auto GenerateFile(std::filesystem::path const& path, std::uintmax_t size, bool is_executable = false) noexcept -> bool; /// \brief Generate a directory in the specified location and fill it with a /// number of randomized files. If the directory exists, it is overwritten. /// The name of each file contains a random number and is prefixed with /// kTreeEntryPrefix (to make tree entry larger for git). Each file contains /// the same random number as in it's name. /// \param path Output path. /// \param entries_count Number of file entries in the directory. /// \return True if the directory is generated properly. [[nodiscard]] static auto GenerateDirectory( std::filesystem::path const& path, std::uintmax_t entries_count) noexcept -> bool; }; #endif // INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/logging/000077500000000000000000000000001516554100600233645ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/logging/log_config.hpp000066400000000000000000000040511516554100600262030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_LOGGING_LOG_CONFIG_HPP #define INCLUDED_SRC_TEST_UTILS_LOGGING_LOG_CONFIG_HPP #include #include #include // std::stoul #include #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" static auto ReadLogLevelFromEnv() -> LogLevel { static constexpr LogLevel kDefaultTestLogLevel = LogLevel::Error; static constexpr LogLevel kMaximumTestLogLevel = LogLevel::Trace; auto log_level{kDefaultTestLogLevel}; auto* log_level_str = std::getenv("LOG_LEVEL_TESTS"); if (log_level_str not_eq nullptr) { try { log_level = static_cast(std::stoul(log_level_str)); } catch (std::exception&) { log_level = kDefaultTestLogLevel; } } switch (log_level) { case LogLevel::Error: case LogLevel::Warning: case LogLevel::Info: case LogLevel::Progress: case LogLevel::Performance: case LogLevel::Debug: case LogLevel::Trace: return log_level; } // log level is out of range return kMaximumTestLogLevel; } [[maybe_unused]] static inline void ConfigureLogging() { LogConfig::SetLogLimit(ReadLogLevelFromEnv()); LogConfig::SetSinks({LogSinkCmdLine::CreateFactory(false /*no color*/)}); } #endif // INCLUDED_SRC_TEST_UTILS_LOGGING_LOG_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/000077500000000000000000000000001516554100600253145ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/EXPRESSIONS000066400000000000000000000154431516554100600270700ustar00rootroot00000000000000{ "run_test": { "doc": [ "Build and run a CC test binary using the provided runner" , "that is expected to also generate remotestdout and remotestderr." ] , "vars": [ "ARCH" , "HOST_ARCH" , "TARGET_ARCH" , "ARCH_DISPATCH" , "TEST_SUMMARY_EXECUTION_PROPERTIES" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "TEST_ENV" , "TIMEOUT_SCALE" , "CC_TEST_LAUNCHER" , "RUNS_PER_TEST" , "name" , "pure C" , "stage" , "srcs" , "private-hdrs" , "private-defines" , "private-cflags" , "private-ldflags" , "defaults-transition" , "deps-transition" , "deps-fieldnames" , "runner" , "runner-data" , "test-args" , "test-data" , "summarizer" , "summary artifacts" , "LINT" ] , "imports": { "artifacts": ["@", "rules", "", "field_artifacts"] , "runfiles": ["@", "rules", "", "field_runfiles"] , "compile-deps": ["@", "rules", "CC", "compile-deps"] , "compile-args-deps": ["@", "rules", "CC", "compile-args-deps"] , "link-deps": ["@", "rules", "CC", "link-deps"] , "link-args-deps": ["@", "rules", "CC", "link-args-deps"] , "cflags-files-deps": ["@", "rules", "CC", "cflags-files-deps"] , "ldflags-files-deps": ["@", "rules", "CC", "ldflags-files-deps"] , "binary": ["@", "rules", "CC", "bin artifact"] , "host transition": ["@", "rules", "transitions", "for host"] , "target properties": ["@", "rules", "transitions", "target properties"] , "stage": ["@", "rules", "", "stage_singleton_field"] , "lint": ["@", "rules", "CC", "lint information"] } , "expression": { "type": "let*" , "bindings": [ [ "cflags-files" , {"type": "CALL_EXPRESSION", "name": "cflags-files-deps"} ] , ["compile-deps", {"type": "CALL_EXPRESSION", "name": "compile-deps"}] , [ "compile-args" , { "type": "++" , "$1": [ { "type": "foreach" , "var": "def" , "range": {"type": "var", "name": "private-defines", "default": []} , "body": {"type": "join", "$1": ["-D", {"type": "var", "name": "def"}]} } , {"type": "var", "name": "private-cflags", "default": []} , {"type": "CALL_EXPRESSION", "name": "compile-args-deps"} ] } ] , [ "ldflags-files" , {"type": "CALL_EXPRESSION", "name": "ldflags-files-deps"} ] , ["link-deps", {"type": "CALL_EXPRESSION", "name": "link-deps"}] , [ "link-args" , { "type": "++" , "$1": [ {"type": "CALL_EXPRESSION", "name": "link-args-deps"} , {"type": "var", "name": "private-ldflags", "default": []} ] } ] , ["binary", {"type": "CALL_EXPRESSION", "name": "binary"}] , [ "lint" , { "type": "if" , "cond": {"type": "var", "name": "LINT"} , "then": { "type": "let*" , "bindings": [ ["hdrs", {"type": "empty_map"}] , [ "lint-deps fieldnames" , ["private-hdrs", "srcs", "private-deps"] ] ] , "body": {"type": "CALL_EXPRESSION", "name": "lint"} } } ] , [ "staged test binary" , { "type": "map_union" , "$1": { "type": "foreach_map" , "range": {"type": "var", "name": "binary"} , "var_val": "binary" , "body": { "type": "singleton_map" , "key": "test" , "value": {"type": "var", "name": "binary"} } } } ] , [ "test-args" , { "type": "singleton_map" , "key": "test-args.json" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": {"type": "var", "name": "test-args", "default": []} } } } ] , [ "test-launcher" , { "type": "singleton_map" , "key": "test-launcher.json" , "value": { "type": "BLOB" , "data": { "type": "json_encode" , "$1": {"type": "var", "name": "CC_TEST_LAUNCHER", "default": []} } } } ] , [ "test-name" , { "type": "join" , "separator": "/" , "$1": [{"type": "var", "name": "stage"}, {"type": "var", "name": "name"}] } ] , [ "test input" , { "type": "map_union" , "$1": [ { "type": "to_subdir" , "subdir": "work" , "$1": {"type": "var", "name": "test-data"} } , {"type": "var", "name": "runner"} , { "type": "var" , "name": "runner-data" , "default": {"type": "empty_map"} } , {"type": "var", "name": "test-args"} , {"type": "var", "name": "test-launcher"} , {"type": "var", "name": "staged test binary"} ] } ] , [ "target properties" , {"type": "CALL_EXPRESSION", "name": "target properties"} ] ] , "body": { "type": "let*" , "bindings": [ [ "test-results" , { "type": "ACTION" , "outs": [ "result" , "stdout" , "stderr" , "remotestdout" , "remotestderr" , "time-start" , "time-stop" , "pwd" ] , "inputs": {"type": "var", "name": "test input"} , "cmd": ["./runner"] , "env": { "type": "var" , "name": "TEST_ENV" , "default": {"type": "empty_map"} } , "may_fail": ["test"] , "fail_message": { "type": "join" , "$1": ["CC test ", {"type": "var", "name": "test-name"}, " failed"] } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} , "execution properties": {"type": "var", "name": "target properties"} } ] , [ "runfiles" , { "type": "singleton_map" , "key": {"type": "var", "name": "name"} , "value": {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} } ] ] , "body": { "type": "RESULT" , "artifacts": {"type": "var", "name": "test-results"} , "runfiles": {"type": "var", "name": "runfiles"} , "provides": {"type": "env", "vars": ["lint"]} } } } } } just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/RULES000066400000000000000000000175331516554100600261420ustar00rootroot00000000000000{ "CC test": { "doc": ["A test written in C++, depending on remote execution"] , "tainted": ["test"] , "target_fields": ["srcs", "private-hdrs", "private-deps", "data"] , "string_fields": [ "name" , "args" , "stage" , "pure C" , "private-defines" , "private-cflags" , "private-ldflags" , "compatible remote" ] , "config_vars": [ "ARCH" , "HOST_ARCH" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "TEST_ENV" , "CC_TEST_LAUNCHER" , "TEST_COMPATIBLE_REMOTE" , "LINT" ] , "implicit": { "defaults": [["@", "rules", "CC", "defaults"]] , "runner": ["test_runner.py"] , "just": [["", "tool-under-test"]] } , "field_doc": { "name": [ "The name of the test" , "" , "Used to name the test binary as well as for staging the test result" ] , "args": ["Command line arguments for the test binary"] , "srcs": ["The sources of the test binary"] , "private-hdrs": [ "Any additional header files that need to be present when compiling" , "the test binary." ] , "private-defines": [ "List of defines set for source files local to this target." , "Each list entry will be prepended by \"-D\"." ] , "private-cflags": ["List of compile flags set for source files local to this target."] , "private-ldflags": [ "Additional linker flags for linking external libraries (not built" , "by this tool, typically system libraries)." ] , "stage": [ "The logical location of all header and source files." , "Individual directory components are joined with \"/\"." ] , "pure C": [ "If non-empty, compile as C sources rather than C++ sources." , "In particular, CC is used to compile rather than CXX (or their" , "respective defaults)." ] , "data": ["Any files the test binary needs access to when running"] } , "config_doc": { "CC": ["The name of the C compiler to be used."] , "CXX": ["The name of the C++ compiler to be used."] , "CFLAGS": [ "The flags for CC to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX" , "taken from the [\"CC\", \"defaults\"] target" ] , "ENV": ["The environment for any action generated."] , "TEST_ENV": ["The environment for executing the test runner."] , "CC_TEST_LAUNCHER": [ "List of strings representing the launcher that is prepend to the" , "command line for running the test binary." ] , "TEST_COMPATIBLE_REMOTE": [ "If true, provide compatible remote execution and set COMPATIBLE in the" , " test environment; otherwise, provide a native remote execution" , "do not modify the environment" ] } , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its" , "\"private-deps\" argument, this gives an easy way of defining test" , "suites." ] , "imports": { "artifacts": ["@", "rules", "", "field_artifacts"] , "runfiles": ["@", "rules", "", "field_runfiles"] , "host transition": ["@", "rules", "transitions", "for host"] , "stage": ["@", "rules", "", "stage_singleton_field"] , "run_test": "run_test" } , "config_transitions": { "defaults": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-hdrs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "srcs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "data": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "just": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ [ "name" , { "type": "assert_non_empty" , "msg": "A non-empty name has to be provided for binaries" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["pure C", {"type": "FIELD", "name": "pure C"}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , ["host-trans", {"type": "CALL_EXPRESSION", "name": "host transition"}] , [ "srcs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "srcs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "private-hdrs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , ["defaults-transition", {"type": "var", "name": "host-trans"}] , ["deps-transition", {"type": "var", "name": "host-trans"}] , ["deps-fieldnames", ["private-deps", "defaults"]] , ["transition", {"type": "CALL_EXPRESSION", "name": "host transition"}] , ["fieldname", "runner"] , ["location", "runner"] , ["runner", {"type": "CALL_EXPRESSION", "name": "stage"}] , ["fieldname", "just"] , ["just", {"type": "CALL_EXPRESSION", "name": "artifacts"}] , [ "compatible-remote" , { "type": "singleton_map" , "key": "compatible-remote.json" , "value": { "type": "BLOB" , "data": { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": "true" , "else": "false" } } } ] , [ "runner-data" , { "type": "map_union" , "$1": [ {"type": "var", "name": "compatible-remote"} , { "type": "to_subdir" , "subdir": "staged" , "$1": {"type": "var", "name": "just"} } ] } ] , ["test-args", {"type": "FIELD", "name": "args", "default": []}] , [ "test-data" , { "type": "let*" , "bindings": [ ["fieldname", "data"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "runfiles"} , {"type": "CALL_EXPRESSION", "name": "artifacts"} ] } } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "run_test"} } } } just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/TARGETS000066400000000000000000000000031516554100600263410ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/bazel_action_creator.hpp000066400000000000000000000122731516554100600322030ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_ACTION_CREATOR_HPP #define INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_ACTION_CREATOR_HPP #include // std::transform, std::copy #include #include #include #include #include #include #include #include #include "google/protobuf/repeated_ptr_field.h" #include "gsl/gsl" #include "src/buildtool/common/artifact_blob.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" [[nodiscard]] static inline auto CreateAction( std::string const& instance_name, std::vector const& args, std::map const& env_vars, std::map const& properties, HashFunction hash_function) noexcept -> std::unique_ptr { auto platform = std::make_unique(); for (auto const& [name, value] : properties) { bazel_re::Platform_Property property; property.set_name(name); property.set_value(value); *(platform->add_properties()) = property; } std::unordered_set blobs; bazel_re::Command cmd; cmd.set_allocated_platform(platform.release()); std::copy( args.begin(), args.end(), pb::back_inserter(cmd.mutable_arguments())); std::transform(std::begin(env_vars), std::end(env_vars), pb::back_inserter(cmd.mutable_environment_variables()), [](auto const& name_value) { bazel_re::Command_EnvironmentVariable env_var_message; env_var_message.set_name(name_value.first); env_var_message.set_value(name_value.second); return env_var_message; }); auto const cmd_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, cmd.SerializeAsString()); if (not cmd_blob.has_value()) { return nullptr; } blobs.emplace(*cmd_blob); bazel_re::Directory const empty_dir; auto const dir_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::Tree, empty_dir.SerializeAsString()); if (not dir_blob.has_value()) { return nullptr; } blobs.emplace(*dir_blob); bazel_re::Action action; (*action.mutable_command_digest()) = ArtifactDigestFactory::ToBazel(cmd_blob->GetDigest()); action.set_do_not_cache(false); (*action.mutable_input_root_digest()) = ArtifactDigestFactory::ToBazel(dir_blob->GetDigest()); auto const action_blob = ArtifactBlob::FromMemory( hash_function, ObjectType::File, action.SerializeAsString()); if (not action_blob.has_value()) { return nullptr; } blobs.emplace(*action_blob); auto auth_config = TestAuthConfig::ReadFromEnvironment(); if (not auth_config) { return nullptr; } auto remote_config = TestRemoteConfig::ReadFromEnvironment(); if (not remote_config or not remote_config->remote_address) { return nullptr; } RetryConfig retry_config{}; // default retry config BazelCapabilitiesClient capabilities(remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config); BazelCasClient cas_client(remote_config->remote_address->host, remote_config->remote_address->port, &*auth_config, &retry_config, &capabilities, /*temp_space=*/nullptr); if (cas_client.BatchUpdateBlobs(instance_name, blobs) == blobs.size()) { return std::make_unique( ArtifactDigestFactory::ToBazel(action_blob->GetDigest())); } return nullptr; } #endif // INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_ACTION_CREATOR_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/main-remote-execution.cpp000066400000000000000000000047431516554100600322460ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #define CATCH_CONFIG_RUNNER #include #include #include #include #include "catch2/catch_session.hpp" #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "test/utils/logging/log_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" namespace { void wait_for_grpc_to_shutdown() { // grpc_shutdown_blocking(); // not working std::this_thread::sleep_for(std::chrono::seconds(1)); } /// \brief Configure remote execution from test environment. In case the /// environment variable is malformed, we write a message and stop execution. /// \returns true If remote execution was successfully configured. void ConfigureRemoteExecution() { // Ensure authentication config is available if (not TestAuthConfig::ReadFromEnvironment()) { std::exit(EXIT_FAILURE); } auto remote_config = TestRemoteConfig::ReadFromEnvironment(); if (not remote_config or remote_config->remote_address == std::nullopt) { std::exit(EXIT_FAILURE); } } } // namespace auto main(int argc, char* argv[]) -> int { ConfigureLogging(); ConfigureRemoteExecution(); /** * The current implementation of libgit2 uses pthread_key_t incorrectly * on POSIX systems to handle thread-specific data, which requires us to * explicitly make sure the main thread is the first one to call * git_libgit2_init. Future versions of libgit2 will hopefully fix this. */ GitContext::Create(); int result = Catch::Session().run(argc, argv); // valgrind fails if we terminate before grpc's async shutdown threads exit wait_for_grpc_to_shutdown(); return result; } just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/test_auth_config.hpp000066400000000000000000000036171516554100600313610ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_AUTH_CONFIG_HPP #define INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_AUTH_CONFIG_HPP #include #include #include #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/test_env.hpp" class TestAuthConfig final { public: [[nodiscard]] static auto ReadFromEnvironment() noexcept -> std::optional { Auth::TLS::Builder tls_builder; auto config = tls_builder.SetCACertificate(ReadTLSAuthCACertFromEnv()) .SetClientCertificate(ReadTLSAuthClientCertFromEnv()) .SetClientKey(ReadTLSAuthClientKeyFromEnv()) .Build(); if (config) { if (*config) { // correctly configured TLS/SSL certification return *std::move(*config); } // given TLS certificates are invalid Logger::Log(LogLevel::Error, config->error()); return std::nullopt; } return Auth{}; // no TLS certificates provided } }; #endif // INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_AUTH_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/test_remote_config.hpp000066400000000000000000000031661516554100600317120ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_REMOTE_CONFIG_HPP #define INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_REMOTE_CONFIG_HPP #include #include #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/test_env.hpp" class TestRemoteConfig final { public: [[nodiscard]] static auto ReadFromEnvironment() noexcept -> std::optional { RemoteExecutionConfig::Builder builder; auto config = builder.SetRemoteAddress(ReadRemoteAddressFromEnv()) .SetPlatformProperties(ReadPlatformPropertiesFromEnv()) .Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } }; #endif // INCLUDED_SRC_TEST_UTILS_REMOTE_EXECUTION_TEST_REMOTE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/remote_execution/test_runner.py000077500000000000000000000072401516554100600302440ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import time time_start: float = time.time() time_stop: float = 0 result: str = "UNKNOWN" stderr: str = "" stdout: str = "" def dump_results() -> None: with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) with open("pwd", "w") as f: f.write("%s\n" % (os.getcwd(), )) dump_results() TEMP_DIR = os.path.realpath("scratch") os.makedirs(TEMP_DIR, exist_ok=True) WORK_DIR = os.path.realpath("work") os.makedirs(WORK_DIR, exist_ok=True) REMOTE_DIR = os.path.realpath("remote") os.makedirs(REMOTE_DIR, exist_ok=True) REMOTE_INFO = os.path.join(REMOTE_DIR, "info.json") REMOTE_LBR = os.path.join(REMOTE_DIR, "build-root") if os.path.exists(REMOTE_INFO): print(f"Warning: removing unexpected info file {REMOTE_INFO}") os.remove(REMOTE_INFO) PATH = subprocess.run( ["env", "--", "sh", "-c", "echo -n $PATH"], stdout=subprocess.PIPE, ).stdout.decode('utf-8') remote_cmd = [ "./staged/bin/just", "execute", "-L", json.dumps(["env", "PATH=" + PATH]), "--info-file", REMOTE_INFO, "--local-build-root", REMOTE_LBR, "--log-limit", "6", "--plain-log", ] with open("compatible-remote.json") as f: compatible = json.load(f) if compatible: remote_cmd.append("--compatible") remotestdout = open("remotestdout", "w") remotestderr = open("remotestderr", "w") remote_proc = subprocess.Popen( remote_cmd, stdout=remotestdout, stderr=remotestderr, ) timeout: int = 30 while not os.path.exists(REMOTE_INFO): if timeout == 0: result = "FAIL" stdout = "Failed to start execution service" remote_proc.terminate() dump_results() exit(1) timeout -= 1 time.sleep(1) with open(REMOTE_INFO) as f: info = json.load(f) REMOTE_EXECUTION_ADDRESS = "%s:%d" % (info["interface"], info["port"]) ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR, TMPDIR=TEMP_DIR, REMOTE_EXECUTION_ADDRESS=REMOTE_EXECUTION_ADDRESS) if compatible: ENV["COMPATIBLE"] = "YES" elif "COMPATIBLE" in ENV: del ENV["COMPATIBLE"] for k in ["TLS_CA_CERT", "TLS_CLIENT_CERT", "TLS_CLIENT_KEY"]: if k in ENV: del ENV[k] with open('test-launcher.json') as f: test_launcher = json.load(f) with open('test-args.json') as f: test_args = json.load(f) time_start = time.time() ret = subprocess.run(test_launcher + ["../test"] + test_args, cwd=WORK_DIR, env=ENV, capture_output=True) time_stop = time.time() result = "PASS" if ret.returncode == 0 else "FAIL" stdout = ret.stdout.decode("utf-8") stderr = ret.stderr.decode("utf-8") remote_proc.terminate() rout, rerr = remote_proc.communicate() dump_results() if result != "PASS": exit(1) just-buildsystem-justbuild-b1fb5fa/test/utils/run_test_server.py000066400000000000000000000030151516554100600255400ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import signal from http.server import SimpleHTTPRequestHandler as HTTPHandler from http.server import HTTPServer as HTTPServer from typing import Any httpd = None # handle interrupts gracefully, i.e., shutdown the server and exit def RecvSig(*_: Any) -> None: if not httpd is None: # cleanup httpd.server_close() sys.exit(0) if __name__ == "__main__": # set calback for usual terminating signals signal.signal(signal.SIGHUP, RecvSig) signal.signal(signal.SIGINT, RecvSig) signal.signal(signal.SIGTERM, RecvSig) # set up server args hostname = "127.0.0.1" # setup server obj with HTTPServer((hostname, 0), HTTPHandler) as httpd: # print port number socket_info = httpd.socket.getsockname() with open(sys.argv[1], "w") as f: f.write("%d" % (socket_info[1],)) # run server httpd.serve_forever() just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/000077500000000000000000000000001516554100600246025ustar00rootroot00000000000000just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/RULES000066400000000000000000000176141516554100600254300ustar00rootroot00000000000000{ "CC test": { "doc": ["A test written in C++, depending on serve service and remote execution"] , "tainted": ["test"] , "target_fields": ["srcs", "private-hdrs", "private-deps", "data"] , "string_fields": [ "name" , "args" , "stage" , "pure C" , "private-defines" , "private-cflags" , "private-ldflags" , "compatible remote" ] , "config_vars": [ "ARCH" , "HOST_ARCH" , "CC" , "CXX" , "CFLAGS" , "CXXFLAGS" , "ADD_CFLAGS" , "ADD_CXXFLAGS" , "ENV" , "TEST_ENV" , "CC_TEST_LAUNCHER" , "TEST_COMPATIBLE_REMOTE" , "LINT" ] , "implicit": { "defaults": [["@", "rules", "CC", "defaults"]] , "runner": ["test_runner.py"] , "just": [["", "tool-under-test"]] } , "field_doc": { "name": [ "The name of the test" , "" , "Used to name the test binary as well as for staging the test result" ] , "args": ["Command line arguments for the test binary"] , "srcs": ["The sources of the test binary"] , "private-hdrs": [ "Any additional header files that need to be present when compiling" , "the test binary." ] , "private-defines": [ "List of defines set for source files local to this target." , "Each list entry will be prepended by \"-D\"." ] , "private-cflags": ["List of compile flags set for source files local to this target."] , "private-ldflags": [ "Additional linker flags for linking external libraries (not built" , "by this tool, typically system libraries)." ] , "stage": [ "The logical location of all header and source files." , "Individual directory components are joined with \"/\"." ] , "pure C": [ "If non-empty, compile as C sources rather than C++ sources." , "In particular, CC is used to compile rather than CXX (or their" , "respective defaults)." ] , "data": ["Any files the test binary needs access to when running"] } , "config_doc": { "CC": ["The name of the C compiler to be used."] , "CXX": ["The name of the C++ compiler to be used."] , "CFLAGS": [ "The flags for CC to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "CXXFLAGS": [ "The flags for CXX to be used instead of the default ones" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CFLAGS": [ "The flags to add to the default ones for CC" , "taken from the [\"CC\", \"defaults\"] target" ] , "ADD_CXXFLAGS": [ "The flags to add to the default ones for CXX" , "taken from the [\"CC\", \"defaults\"] target" ] , "ENV": ["The environment for any action generated."] , "TEST_ENV": ["The environment for executing the test runner."] , "CC_TEST_LAUNCHER": [ "List of strings representing the launcher that is prepend to the" , "command line for running the test binary." ] , "TEST_COMPATIBLE_REMOTE": [ "If true, provide compatible remote execution and set COMPATIBLE in the" , " test environment; otherwise, provide a native remote execution" , "do not modify the environment" ] } , "artifacts_doc": [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" , " generating test reports." , "stdout/stderr: Any output the invocation of the test binary produced on" , " the respective file descriptor" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." , "As the built-in \"install\" rule only takes the runfiles of its" , "\"private-deps\" argument, this gives an easy way of defining test" , "suites." ] , "imports": { "artifacts": ["@", "rules", "", "field_artifacts"] , "runfiles": ["@", "rules", "", "field_runfiles"] , "host transition": ["@", "rules", "transitions", "for host"] , "stage": ["@", "rules", "", "stage_singleton_field"] , "run_test": ["@", "rules", "CC/test", "run_test"] } , "config_transitions": { "defaults": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "private-hdrs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "srcs": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "data": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "just": [{"type": "CALL_EXPRESSION", "name": "host transition"}] , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}] } , "expression": { "type": "let*" , "bindings": [ [ "name" , { "type": "assert_non_empty" , "msg": "A non-empty name has to be provided for binaries" , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} } ] , ["pure C", {"type": "FIELD", "name": "pure C"}] , [ "stage" , { "type": "join" , "separator": "/" , "$1": {"type": "FIELD", "name": "stage"} } ] , ["host-trans", {"type": "CALL_EXPRESSION", "name": "host transition"}] , [ "srcs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "srcs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , [ "private-hdrs" , { "type": "to_subdir" , "subdir": {"type": "var", "name": "stage"} , "$1": { "type": "let*" , "bindings": [ ["fieldname", "private-hdrs"] , ["transition", {"type": "var", "name": "host-trans"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} } } ] , ["defaults-transition", {"type": "var", "name": "host-trans"}] , ["deps-transition", {"type": "var", "name": "host-trans"}] , ["deps-fieldnames", ["private-deps", "defaults"]] , ["transition", {"type": "CALL_EXPRESSION", "name": "host transition"}] , ["fieldname", "runner"] , ["location", "runner"] , ["runner", {"type": "CALL_EXPRESSION", "name": "stage"}] , ["fieldname", "just"] , ["just", {"type": "CALL_EXPRESSION", "name": "artifacts"}] , [ "compatible-remote" , { "type": "singleton_map" , "key": "compatible-remote.json" , "value": { "type": "BLOB" , "data": { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": "true" , "else": "false" } } } ] , [ "runner-data" , { "type": "map_union" , "$1": [ {"type": "var", "name": "compatible-remote"} , { "type": "to_subdir" , "subdir": "staged" , "$1": {"type": "var", "name": "just"} } ] } ] , ["test-args", {"type": "FIELD", "name": "args", "default": []}] , [ "test-data" , { "type": "let*" , "bindings": [ ["fieldname", "data"] , ["transition", {"type": "var", "name": "deps-transition"}] ] , "body": { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "runfiles"} , {"type": "CALL_EXPRESSION", "name": "artifacts"} ] } } ] ] , "body": {"type": "CALL_EXPRESSION", "name": "run_test"} } } } just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/TARGETS000066400000000000000000000000031516554100600256270ustar00rootroot00000000000000{} just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/main-serve.cpp000066400000000000000000000114171516554100600273600ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #define CATCH_CONFIG_RUNNER #include #include #include #include #include #include #include #include "catch2/catch_session.hpp" #include "fmt/core.h" #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/file_system/git_context.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "test/utils/logging/log_config.hpp" #include "test/utils/serve_service/test_serve_config.hpp" #include "test/utils/shell_quoting.hpp" namespace { auto const kBundlePath = std::string{"test/buildtool/file_system/data/test_repo.bundle"}; auto const kBundlePathSymlinks = std::string{"test/buildtool/file_system/data/test_repo_symlinks.bundle"}; void wait_for_grpc_to_shutdown() { // grpc_shutdown_blocking(); // not working std::this_thread::sleep_for(std::chrono::seconds(1)); } [[nodiscard]] auto CloneRepo(std::filesystem::path const& repo_path, std::string const& bundle, bool is_bare = false) noexcept -> bool { auto cmd = fmt::format("git clone {}{} {}", is_bare ? "--bare " : "", QuoteForShell(bundle), QuoteForShell(repo_path.string())); return (std::system(cmd.c_str()) == 0); } [[nodiscard]] auto CreateServeTestRepo(std::filesystem::path const& repo_path, std::string const& bundle, bool is_bare = false) noexcept -> bool { if (not CloneRepo(repo_path, bundle, is_bare)) { return false; } auto cmd = fmt::format("git --git-dir={} --work-tree={} checkout master", QuoteForShell(is_bare ? repo_path.string() : (repo_path / ".git").string()), QuoteForShell(repo_path.string())); return (std::system(cmd.c_str()) == 0); } /// \brief Configure serve service from test environment. In case the /// environment variable is malformed, we write a message and stop execution. /// The availability of the serve service known repositories (from known test /// bundles) is also ensured. /// \returns true If serve service was successfully configured. [[nodiscard]] auto ConfigureServeService() -> bool { // just serve shares here compatibility and authentication args with // remote execution, so no need to do those again // Ensure the config can be read from the environment auto config = TestServeConfig::ReadFromEnvironment(); if (not config or not config->remote_address) { return false; } // now actually populate the serve repositories, one bare and one non-bare if (config->known_repositories.size() != 2) { Logger::Log(LogLevel::Error, "Expected 2 serve repositories in test env."); std::exit(EXIT_FAILURE); } auto const& bare_repo = config->known_repositories[0]; auto const& nonbare_repo = config->known_repositories[1]; if (not CreateServeTestRepo(bare_repo, kBundlePath, /*is_bare=*/true) or not CreateServeTestRepo( nonbare_repo, kBundlePathSymlinks, /*is_bare=*/false)) { Logger::Log(LogLevel::Error, "Failed to setup serve service repositories."); std::exit(EXIT_FAILURE); } return true; } } // namespace auto main(int argc, char* argv[]) -> int { ConfigureLogging(); // Setup of serve service, including known repositories. if (not ConfigureServeService()) { return EXIT_FAILURE; } /** * The current implementation of libgit2 uses pthread_key_t incorrectly * on POSIX systems to handle thread-specific data, which requires us to * explicitly make sure the main thread is the first one to call * git_libgit2_init. Future versions of libgit2 will hopefully fix this. */ GitContext::Create(); int result = Catch::Session().run(argc, argv); // valgrind fails if we terminate before grpc's async shutdown threads exit wait_for_grpc_to_shutdown(); return result; } just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/test_runner.py000077500000000000000000000116121516554100600275300ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import time time_start: float = time.time() time_stop: float = 0 result: str = "UNKNOWN" stderr: str = "" stdout: str = "" def dump_results() -> None: with open("result", "w") as f: f.write("%s\n" % (result, )) with open("time-start", "w") as f: f.write("%d\n" % (time_start, )) with open("time-stop", "w") as f: f.write("%d\n" % (time_stop, )) with open("stdout", "w") as f: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) with open("pwd", "w") as f: f.write("%s\n" % (os.getcwd(), )) dump_results() TEMP_DIR = os.path.abspath(os.path.realpath("scratch")) os.makedirs(TEMP_DIR, exist_ok=True) WORK_DIR = os.path.abspath(os.path.realpath("work")) os.makedirs(WORK_DIR, exist_ok=True) REMOTE_DIR = os.path.abspath(os.path.realpath("remote")) os.makedirs(REMOTE_DIR, exist_ok=True) SERVE_CONFIG_FILE = os.path.realpath("just-servec") # Set known serve repository roots under TEST_TMPDIR TEST_SERVE_REPO_1 = os.path.join(TEMP_DIR, "test_serve_repo_1") if os.path.exists(TEST_SERVE_REPO_1): os.remove(TEST_SERVE_REPO_1) TEST_SERVE_REPO_2 = os.path.join(TEMP_DIR, "test_serve_repo_2") if os.path.exists(TEST_SERVE_REPO_2): os.remove(TEST_SERVE_REPO_2) SERVE_REPOSITORIES = ";".join([TEST_SERVE_REPO_1, TEST_SERVE_REPO_2]) REMOTE_SERVE_INFO = os.path.join(REMOTE_DIR, "info_serve.json") SERVE_LBR = os.path.join(REMOTE_DIR, "serve-build-root") if os.path.exists(REMOTE_SERVE_INFO): print(f"Warning: removing unexpected info file {REMOTE_SERVE_INFO}") os.remove(REMOTE_SERVE_INFO) # Run 'just serve' PATH = subprocess.run( ["env", "--", "sh", "-c", "echo -n $PATH"], stdout=subprocess.PIPE, ).stdout.decode('utf-8') compatible: bool = False with open("compatible-remote.json") as f: s = json.load(f) if s: compatible = True with open(SERVE_CONFIG_FILE, "w") as f: f.write( json.dumps({ "repositories": [{ "root": "system", "path": TEST_SERVE_REPO_1 }, { "root": "system", "path": TEST_SERVE_REPO_2 }], "logging": { "limit": 6, "plain": True }, "remote service": { "info file": { "root": "system", "path": REMOTE_SERVE_INFO } }, "local build root": { "root": "system", "path": SERVE_LBR }, "execution endpoint": { "compatible": compatible }, "build": { "local launcher": ["env", "PATH=" + PATH] } })) serve_cmd = ["./staged/bin/just", "serve", SERVE_CONFIG_FILE] servestdout = open("servestdout", "w") servestderr = open("servestderr", "w") serve_proc = subprocess.Popen( serve_cmd, stdout=servestdout, stderr=servestderr, ) timeout: int = 30 while not os.path.exists(REMOTE_SERVE_INFO): if timeout == 0: result = "FAIL" stdout = "Failed to start serve service" serve_proc.terminate() dump_results() exit(1) timeout -= 1 time.sleep(1) with open(REMOTE_SERVE_INFO) as f: serve_info = json.load(f) REMOTE_SERVE_ADDRESS = "%s:%d" % (serve_info["interface"], serve_info["port"]) # Setup environment ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR, TMPDIR=TEMP_DIR, REMOTE_SERVE_ADDRESS=REMOTE_SERVE_ADDRESS, SERVE_REPOSITORIES=SERVE_REPOSITORIES) for k in ["TLS_CA_CERT", "TLS_CLIENT_CERT", "TLS_CLIENT_KEY"]: if k in ENV: del ENV[k] with open('test-launcher.json') as f: test_launcher = json.load(f) with open('test-args.json') as f: test_args = json.load(f) # Run the test time_start = time.time() ret = subprocess.run(test_launcher + ["../test"] + test_args, cwd=WORK_DIR, env=ENV, capture_output=True) time_stop = time.time() result = "PASS" if ret.returncode == 0 else "FAIL" stdout = ret.stdout.decode("utf-8") stderr = ret.stderr.decode("utf-8") serve_proc.terminate() sout, serr = serve_proc.communicate() dump_results() if result != "PASS": exit(1) just-buildsystem-justbuild-b1fb5fa/test/utils/serve_service/test_serve_config.hpp000066400000000000000000000031471516554100600310300ustar00rootroot00000000000000// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_SERVE_SERVICE_TEST_SERVE_CONFIG_HPP #define INCLUDED_SRC_TEST_UTILS_SERVE_SERVICE_TEST_SERVE_CONFIG_HPP #include #include #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/utils/cpp/expected.hpp" #include "test/utils/test_env.hpp" class TestServeConfig final { public: [[nodiscard]] static auto ReadFromEnvironment() noexcept -> std::optional { RemoteServeConfig::Builder builder; auto config = builder.SetRemoteAddress(ReadRemoteServeAddressFromEnv()) .SetKnownRepositories(ReadRemoteServeReposFromEnv()) .Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } }; #endif // INCLUDED_SRC_TEST_UTILS_SERVE_SERVICE_TEST_SERVE_CONFIG_HPP just-buildsystem-justbuild-b1fb5fa/test/utils/shell_quoting.hpp000066400000000000000000000024771516554100600253360ustar00rootroot00000000000000// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include [[nodiscard]] static inline auto QuoteForShell(std::string const& input) -> std::string { // To correctly quote a string for shell, replace ever ~'~ char with ~'\''~ // and then put the resulting inside a pair of ~'~s. auto output = std::string(R"(')"); auto start = input.begin(); auto pos = input.find('\''); while (pos != std::string::npos) { output.append( start, start + static_cast(pos + 1)); output += R"(\'')"; start += static_cast(pos + 1); pos = input.find('\'', pos + 1); } output.append(start, input.end()); output += R"(')"; return output; } just-buildsystem-justbuild-b1fb5fa/test/utils/test_env.hpp000066400000000000000000000067371516554100600243130ustar00rootroot00000000000000// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_TEST_UTILS_TEST_ENV_HPP #define INCLUDED_SRC_TEST_UTILS_TEST_ENV_HPP #include #include #include #include #include #include [[nodiscard]] static inline auto ReadPlatformPropertiesFromEnv() -> std::vector { std::vector properties{}; auto* execution_props = std::getenv("REMOTE_EXECUTION_PROPERTIES"); if (execution_props not_eq nullptr) { std::istringstream pss(std::string{execution_props}); std::string keyval_pair; while (std::getline(pss, keyval_pair, ' ')) { properties.emplace_back(keyval_pair); } } return properties; } [[nodiscard]] static inline auto ReadCompatibilityFromEnv() noexcept -> std::optional { try { return std::getenv("COMPATIBLE") != nullptr; } catch (...) { return std::nullopt; } } [[nodiscard]] static inline auto ReadRemoteAddressFromEnv() -> std::optional { auto* execution_address = std::getenv("REMOTE_EXECUTION_ADDRESS"); return execution_address == nullptr ? std::nullopt : std::make_optional(std::string{execution_address}); } [[nodiscard]] static inline auto ReadRemoteServeAddressFromEnv() -> std::optional { auto* serve_address = std::getenv("REMOTE_SERVE_ADDRESS"); return serve_address == nullptr ? std::nullopt : std::make_optional(std::string{serve_address}); } [[nodiscard]] static inline auto ReadTLSAuthCACertFromEnv() -> std::optional { auto* ca_cert = std::getenv("TLS_CA_CERT"); return ca_cert == nullptr ? std::nullopt : std::make_optional(std::filesystem::path(ca_cert)); } [[nodiscard]] static inline auto ReadTLSAuthClientCertFromEnv() -> std::optional { auto* client_cert = std::getenv("TLS_CLIENT_CERT"); return client_cert == nullptr ? std::nullopt : std::make_optional(std::filesystem::path(client_cert)); } [[nodiscard]] static inline auto ReadTLSAuthClientKeyFromEnv() -> std::optional { auto* client_key = std::getenv("TLS_CLIENT_KEY"); return client_key == nullptr ? std::nullopt : std::make_optional(std::filesystem::path(client_key)); } [[nodiscard]] static inline auto ReadRemoteServeReposFromEnv() -> std::vector { std::vector repos{}; auto* serve_repos = std::getenv("SERVE_REPOSITORIES"); if (serve_repos not_eq nullptr) { std::istringstream pss(std::string{serve_repos}); std::string path; while (std::getline(pss, path, ';')) { repos.emplace_back(path); } } return repos; } #endif // INCLUDED_SRC_TEST_UTILS_TEST_ENV_HPP