mirror of
https://github.com/axodotdev/cargo-dist.git
synced 2025-02-17 01:02:02 +08:00
feat: add cross-compilation support
This commit is contained in:
Amos Wenger
Misty De Méo
parent
e491b6a06b
commit
2ea561ca6f
8
.github/workflows/release.yml
vendored
8
.github/workflows/release.yml
vendored
@ -111,6 +111,7 @@ jobs:
|
||||
# - N "local" tasks that build each platform's binaries and platform-specific installers
|
||||
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.container && matrix.container.image || null }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
|
||||
@ -121,6 +122,13 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Install Rust non-interactively if not already installed
|
||||
if: ${{ matrix.container }}
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
- name: Install dist
|
||||
run: ${{ matrix.install_dist.run }}
|
||||
# Get the dist-manifest
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -8,3 +8,6 @@ cargo-dist/wix
|
||||
oranda-debug.log
|
||||
/public
|
||||
cargo-dist/public
|
||||
|
||||
# Samply: <https://crates.io/crates/samply>
|
||||
/profile.json
|
||||
|
||||
5
Cargo.lock
generated
5
Cargo.lock
generated
@ -453,6 +453,7 @@ dependencies = [
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2829,9 +2830,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.14"
|
||||
version = "0.12.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
|
||||
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
||||
|
||||
[[package]]
|
||||
name = "target-spec"
|
||||
|
||||
@ -70,6 +70,7 @@ lazy_static = "1.5.0"
|
||||
current_platform = "0.2.0"
|
||||
color-backtrace = "0.6.1"
|
||||
backtrace = "0.3.74"
|
||||
target-lexicon = "0.12.16"
|
||||
|
||||
[workspace.metadata.release]
|
||||
shared-version = true
|
||||
|
||||
80
Justfile
Normal file
80
Justfile
Normal file
@ -0,0 +1,80 @@
|
||||
|
||||
build-all-plaforms:
|
||||
#!/bin/bash -eux
|
||||
export AXOASSET_XZ_LEVEL=1
|
||||
cargo build
|
||||
./target/debug/dist init --yes
|
||||
./target/debug/dist build --artifacts all --target aarch64-apple-darwinx,x86_64-apple-darwin,x86_64-pc-windows-msvc,x86_64-unknown-linux-musl
|
||||
|
||||
patch-sh-installer:
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const installerUrl = process.env.INSTALLER_DOWNLOAD_URL || 'https://dl.bearcove.cloud/dump/dist-cross';
|
||||
const installerPath = './target/distrib/cargo-dist-installer.sh';
|
||||
|
||||
const content = fs.readFileSync(installerPath, 'utf8');
|
||||
const lines = content.split('\n');
|
||||
let modified = false;
|
||||
|
||||
const newLines = [];
|
||||
for (const line of lines) {
|
||||
if (line.includes('export INSTALLER_DOWNLOAD_URL')) {
|
||||
continue;
|
||||
}
|
||||
if (line.includes('set -u') && !modified) {
|
||||
modified = true;
|
||||
newLines.push(line);
|
||||
newLines.push(`export INSTALLER_DOWNLOAD_URL=${installerUrl} # patched by Justfile in dist repo, using dist_url_override feature`);
|
||||
continue;
|
||||
}
|
||||
newLines.push(line);
|
||||
}
|
||||
|
||||
fs.writeFileSync(installerPath, newLines.join('\n'));
|
||||
|
||||
if (modified) {
|
||||
console.log('\x1b[32m%s\x1b[0m', `✅ ${installerPath} patched successfully!`);
|
||||
console.log('\x1b[36m%s\x1b[0m', `🔗 Pointing to: ${installerUrl}`);
|
||||
} else {
|
||||
console.log('\x1b[31m%s\x1b[0m', '❌ Error: Could not find line with "set -u" in installer script');
|
||||
}
|
||||
|
||||
patch-ps1-installer:
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const installerUrl = process.env.INSTALLER_DOWNLOAD_URL || 'https://dl.bearcove.cloud/dump/dist-cross';
|
||||
const installerPath = './target/distrib/cargo-dist-installer.ps1';
|
||||
|
||||
const content = fs.readFileSync(installerPath, 'utf8');
|
||||
const lines = content.split('\n');
|
||||
let modified = false;
|
||||
|
||||
const newLines = [];
|
||||
for (const line of lines) {
|
||||
if (line.includes('$env:INSTALLER_DOWNLOAD_URL = ')) {
|
||||
continue;
|
||||
}
|
||||
if (line.includes('$app_name = ') && !modified) {
|
||||
modified = true;
|
||||
newLines.push(`$env:INSTALLER_DOWNLOAD_URL = "${installerUrl}" # patched by Justfile in dist repo, using dist_url_override feature`);
|
||||
newLines.push(line);
|
||||
continue;
|
||||
}
|
||||
newLines.push(line);
|
||||
}
|
||||
|
||||
fs.writeFileSync(installerPath, newLines.join('\n'));
|
||||
|
||||
if (modified) {
|
||||
console.log('\x1b[32m%s\x1b[0m', `✅ ${installerPath} patched successfully!`);
|
||||
console.log('\x1b[36m%s\x1b[0m', `🔗 Pointing to: ${installerUrl}`);
|
||||
} else {
|
||||
console.log('\x1b[31m%s\x1b[0m', '❌ Error: Could not find line with "cargo-dist = " in installer script');
|
||||
}
|
||||
|
||||
dump:
|
||||
#!/bin/bash -eux
|
||||
just build-all-plaforms
|
||||
just patch-sh-installer
|
||||
just patch-ps1-installer
|
||||
mc mirror --overwrite ./target/distrib ${DIST_TARGET:-bearcove/dump/dist-cross}
|
||||
@ -4,14 +4,13 @@
|
||||
|
||||
dist's generated CI configuration can be extended in several ways: it can be configured to install extra packages before the build begins, and it's possible to add extra jobs to run at specific lifecycle moments.
|
||||
|
||||
|
||||
## Install extra packages
|
||||
|
||||
> since 0.4.0
|
||||
|
||||
Sometimes, you may need extra packages from the system package manager to be installed before in the builder before dist begins building your software. dist can do this for you by adding the `dependencies` setting to your dist config. When set, the packages you request will be fetched and installed in the step before `build`. Additionally, on macOS, the `cargo build` process will be wrapped in `brew bundle exec` to ensure that your dependencies can be found no matter where Homebrew placed them.
|
||||
|
||||
By default, we run Apple silicon (aarch64) builds for macOS on the `macos-13` runner, which is Intel-based. If your build process needs to link against C libraries from Homebrew using the `dependencies` feature, you will need to switch to an Apple silicon-native runner to ensure that you have access to Apple silicon-native dependencies from Homebrew. You can do this using the [custom runners][custom-runners] feature. Currently, `macos-14` is the oldest (and only) GitHub-provided runner for Apple silicon.
|
||||
By default, we run Apple silicon (aarch64) builds for macOS on the `macos-13` runner, which is Intel-based. If your build process needs to link against C libraries from Homebrew using the `dependencies` feature, you will need to switch to an Apple silicon-native runner to ensure that you have access to Apple silicon-native dependencies from Homebrew. You can do this using the [custom runners][custom-runners] feature. Currently, `macos-14` is the oldest GitHub-provided runner for Apple silicon.
|
||||
|
||||
Sometimes, you may want to make sure your users also have these dependencies available when they install your software. If you use a package manager-based installer, dist has the ability to specify these dependencies. By default, dist will examine your program to try to detect which dependencies it thinks will be necessary. At the moment, [Homebrew][homebrew] is the only supported package manager installer. You can also specify these dependencies manually.
|
||||
|
||||
@ -96,7 +95,9 @@ By default, dist uses the following runners:
|
||||
It's possible to configure alternate runners for these jobs, or runners for targets not natively supported by GitHub actions. To do this, use the [`github-custom-runners`][config-github-custom-runners] configuration setting in your dist config. Here's an example which adds support for Linux (aarch64) using runners from [Buildjet](https://buildjet.com/for-github-actions):
|
||||
|
||||
```toml
|
||||
[workspace.metadata.dist.github-custom-runners]
|
||||
# in `dist-workspace.toml`
|
||||
|
||||
[dist.github-custom-runners]
|
||||
aarch64-unknown-linux-gnu = "buildjet-8vcpu-ubuntu-2204-arm"
|
||||
aarch64-unknown-linux-musl = "buildjet-8vcpu-ubuntu-2204-arm"
|
||||
```
|
||||
@ -104,10 +105,139 @@ aarch64-unknown-linux-musl = "buildjet-8vcpu-ubuntu-2204-arm"
|
||||
In addition to adding support for new targets, some users may find it useful to use this feature to fine-tune their builds for supported targets. For example, some projects may wish to build on a newer Ubuntu runner or alternate Linux distros, or may wish to opt into building for Apple Silicon from a native runner by using the `macos-14` runner. Here's an example which uses `macos-14` for native Apple Silicon builds:
|
||||
|
||||
```toml
|
||||
[workspace.metadata.dist.github-custom-runners]
|
||||
# in `dist-workspace.toml`
|
||||
|
||||
[dist.github-custom-runners]
|
||||
aarch64-apple-darwin = "macos-14"
|
||||
```
|
||||
|
||||
## Cross-compilation
|
||||
|
||||
> since 0.26.0
|
||||
|
||||
dist will transparently use either of:
|
||||
|
||||
* [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild)
|
||||
* [cargo-xwin](https://github.com/rust-cross/cargo-xwin)
|
||||
|
||||
To try and build for the target you specified, from the host you specified.
|
||||
|
||||
dist hardcodes knowledge of which cargo wrappers are better suited for which cross: `cargo-zigbuild`
|
||||
handles `x86_64-unknown-linux-gnu` to `aarch64-unknown-linux-gnu` handsomely, for example.
|
||||
|
||||
So if you ask for `aarch64-unknown-linux-gnu` artifacts, because at the time of this writing
|
||||
there are no free `aarch64` GitHub runners, dist will assume you meant this:
|
||||
|
||||
```toml
|
||||
[dist.github-custom-runners]
|
||||
aarch64-unknown-linux-gnu = "ubuntu-20.04"
|
||||
```
|
||||
|
||||
Which really means this:
|
||||
|
||||
```toml
|
||||
[dist.github-custom-runners.aarch64-unknown-linux-gnu]
|
||||
runner = "ubuntu-20.04"
|
||||
host = "x86_64-unknown-linux-gnu"
|
||||
```
|
||||
|
||||
...since dist knows which platform GitHub's own [runner
|
||||
images](https://github.com/actions/runner-images) are.
|
||||
|
||||
So you really only need to specify the `host` if you use [third-party GitHub Actions
|
||||
runners](https://github.com/neysofu/awesome-github-actions-runners?tab=readme-ov-file#list-of-providers) (Namespace, Buildjet, etc.)
|
||||
|
||||
If you don't specify the host, dist will just assume it's the same platform as
|
||||
the target, which is why this works:
|
||||
|
||||
```toml
|
||||
[dist.github-custom-runners]
|
||||
aarch64-unknown-linux-gnu = "buildjet-8vcpu-ubuntu-2204-arm"
|
||||
```
|
||||
|
||||
Building `aarch64-pc-windows-msvc` binaries from a `x86_64-pc-windows-msvc` runner (like
|
||||
`windows-2019`) is surprisingly hard. But building both binaries from an `x86_64-unknown-linux-gnu`
|
||||
runner is surprisingly easy via `cargo-xwin`
|
||||
|
||||
This will work, eventually:
|
||||
|
||||
```toml
|
||||
# in `dist-workspace.toml`
|
||||
|
||||
[dist]
|
||||
targets = ["x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"]
|
||||
|
||||
[dist.github-custom-runners.x86_64-pc-windows-msvc]
|
||||
runner = "ubuntu-20.04"
|
||||
|
||||
[dist.github-custom-runners.aarch64-pc-windows-msvc]
|
||||
runner = "ubuntu-20.04"
|
||||
```
|
||||
|
||||
...because dist can install `cargo-xwin` via `pip`. However, it will take
|
||||
forever. It's probably best to use a docker image that already has
|
||||
`cargo-xwin` installed, and other dependencies you probably want:
|
||||
|
||||
```toml
|
||||
# in `dist-workspace.toml`
|
||||
|
||||
[dist]
|
||||
targets = ["x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"]
|
||||
|
||||
[dist.github-custom-runners.x86_64-pc-windows-msvc]
|
||||
container = "messense/cargo-xwin"
|
||||
|
||||
[dist.github-custom-runners.aarch64-pc-windows-msvc]
|
||||
container = "messense/cargo-xwin"
|
||||
```
|
||||
|
||||
Which is short for:
|
||||
|
||||
```toml
|
||||
# cut: the rest of the config file
|
||||
|
||||
[dist.github-custom-runners.x86_64-pc-windows-msvc]
|
||||
container = { image = "messense/cargo-xwin", host = "x86_64-unknown-linux-gnu" }
|
||||
|
||||
# etc.
|
||||
```
|
||||
|
||||
...but unfortunately, GitHub Actions's "run workflows in container" feature doesn't
|
||||
support emulation yet. We'd have to set up qemu, run docker manually, etc. — which
|
||||
dist doesn't do as of now. So the `host` just defaults to `x86_64-unknown-linux-gnu`
|
||||
right now, because that's all the GitHub runners support anywyay.
|
||||
|
||||
So, because we're only specifying one feature, it's probably easier to just write this:
|
||||
|
||||
```toml
|
||||
[dist]
|
||||
targets = ["x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"]
|
||||
|
||||
[dist.github-custom-runners]
|
||||
x86_64-pc-windows-msvc.container = "messense/cargo-xwin"
|
||||
aarch64-pc-windows-msvc.container = "messense/cargo-xwin"
|
||||
|
||||
# (yes, that /is/ valid TOML)
|
||||
```
|
||||
|
||||
Note that you can use containers for non-cross reasons: maybe you want your binaries to be
|
||||
compatible with really old versions of glibc, older than Ubuntu 20.04: in this case, you
|
||||
can do something like:
|
||||
|
||||
```toml
|
||||
[dist.github-custom-runners.x86_64-unknown-linux-gnu]
|
||||
container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
|
||||
[dist.github-custom-runners.aarch64-unknown-linux-gnu]
|
||||
container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
```
|
||||
|
||||
Note that here, the host triple for those container images is overriden to be `x86_64-unknown-linux-musl`, because dist itself (which must run in the container) might be using a too-recent version of glibc.
|
||||
|
||||
Because of dist's cross-compilation support, if you have both `cargo-zigbuild` and `cargo-xwin`
|
||||
installed on a macOS machine, you can build pretty much every target dist supports, by running
|
||||
`dist build --artifacts all` — in fact, this is used to develop dist itself!
|
||||
|
||||
### Build and upload artifacts on every pull request
|
||||
|
||||
> since 0.3.0
|
||||
|
||||
@ -26,7 +26,7 @@ schemars.workspace = true
|
||||
semver.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
target-lexicon.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
insta.workspace = true
|
||||
|
||||
@ -9,19 +9,26 @@
|
||||
//! The root type of the schema is [`DistManifest`][].
|
||||
|
||||
pub mod macros;
|
||||
pub use target_lexicon;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::{collections::BTreeMap, str::FromStr};
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use target_lexicon::Triple;
|
||||
|
||||
declare_strongly_typed_string! {
|
||||
/// A rust target-triple (e.g. "x86_64-pc-windows-msvc")
|
||||
pub struct TargetTriple => &TargetTripleRef;
|
||||
/// A rustc-like target triple/tuple (e.g. "x86_64-pc-windows-msvc")
|
||||
pub struct TripleName => &TripleNameRef;
|
||||
}
|
||||
|
||||
impl TargetTripleRef {
|
||||
impl TripleNameRef {
|
||||
/// Parse as a [`Triple`]
|
||||
pub fn parse(&self) -> Result<Triple, <Triple as FromStr>::Err> {
|
||||
Triple::from_str(self.as_str())
|
||||
}
|
||||
|
||||
/// Returns true if this target triple contains the word "musl"
|
||||
pub fn is_musl(&self) -> bool {
|
||||
self.0.contains("musl")
|
||||
@ -70,12 +77,18 @@ impl TargetTripleRef {
|
||||
self.0.contains("windows-msvc")
|
||||
}
|
||||
}
|
||||
|
||||
declare_strongly_typed_string! {
|
||||
/// The name of a Github Actions Runner, like `ubuntu-20.04` or `macos-13`
|
||||
pub struct GithubRunner => &GithubRunnerRef;
|
||||
|
||||
/// A container image, like `quay.io/pypa/manylinux_2_28_x86_64`
|
||||
pub struct ContainerImage => &ContainerImageRef;
|
||||
}
|
||||
|
||||
/// Github runners configuration (which github image/container should be used
|
||||
/// to build which target).
|
||||
pub type GithubRunners = BTreeMap<TripleName, GithubRunnerConfig>;
|
||||
|
||||
impl GithubRunnerRef {
|
||||
/// Does the runner name contain the word "buildjet"?
|
||||
pub fn is_buildjet(&self) -> bool {
|
||||
@ -83,6 +96,31 @@ impl GithubRunnerRef {
|
||||
}
|
||||
}
|
||||
|
||||
/// A value or just a string
|
||||
///
|
||||
/// This allows us to have a simple string-based version of a config while still
|
||||
/// allowing for a more advanced version to exist.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringLikeOr<S, T> {
|
||||
/// They gave the simple string-like value (see `declare_strongly_typed_string!`)
|
||||
StringLike(S),
|
||||
/// They gave a more interesting value
|
||||
Val(T),
|
||||
}
|
||||
|
||||
impl<S, T> StringLikeOr<S, T> {
|
||||
/// Constructs a new `StringLikeOr` from the string-like value `s`
|
||||
pub fn from_s(s: S) -> Self {
|
||||
Self::StringLike(s)
|
||||
}
|
||||
|
||||
/// Constructs a new `StringLikeOr` from the more interesting value `t`
|
||||
pub fn from_t(t: T) -> Self {
|
||||
Self::Val(t)
|
||||
}
|
||||
}
|
||||
|
||||
/// A local system path on the machine dist was run.
|
||||
///
|
||||
/// This is a String because when deserializing this may be a path format from a different OS!
|
||||
@ -249,7 +287,7 @@ pub struct AssetInfo {
|
||||
/// * length 0: not a meaningful question, maybe some static file
|
||||
/// * length 1: typical of binaries
|
||||
/// * length 2+: some kind of universal binary
|
||||
pub target_triples: Vec<TargetTriple>,
|
||||
pub target_triples: Vec<TripleName>,
|
||||
/// the linkage of this Asset
|
||||
pub linkage: Option<Linkage>,
|
||||
}
|
||||
@ -284,7 +322,7 @@ pub struct GithubMatrix {
|
||||
/// define each task manually rather than doing cross-product stuff
|
||||
#[serde(default)]
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub include: Vec<GithubMatrixEntry>,
|
||||
pub include: Vec<GithubLocalJobConfig>,
|
||||
}
|
||||
|
||||
impl GithubMatrix {
|
||||
@ -296,29 +334,149 @@ impl GithubMatrix {
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry for a github matrix
|
||||
declare_strongly_typed_string! {
|
||||
/// A bit of shell script to install brew/apt/chocolatey/etc. packages
|
||||
pub struct PackageInstallScript => &PackageInstallScriptRef;
|
||||
}
|
||||
|
||||
/// The version of `GithubRunnerConfig` that's deserialized from the config file: it
|
||||
/// has optional fields that are computed later.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct GithubMatrixEntry {
|
||||
/// Targets to build for
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub targets: Option<Vec<TargetTriple>>,
|
||||
/// Github Runner to user
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub struct GithubRunnerConfigInput {
|
||||
/// GHA's `runs-on` key: Github Runner image to use, see <https://github.com/actions/runner-images>
|
||||
/// and <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/choosing-the-runner-for-a-job>
|
||||
///
|
||||
/// This is not necessarily a well-known runner, it could be something self-hosted, it
|
||||
/// could be from BuildJet, Namespace, etc.
|
||||
///
|
||||
/// If not specified, `container` has to be set.
|
||||
pub runner: Option<GithubRunner>,
|
||||
|
||||
/// Host triple of the runner (well-known, custom, or best guess).
|
||||
/// If the runner is one of GitHub's official runner images, the platform
|
||||
/// is hardcoded. If it's custom, then we have a `target_triple => runner` in the config
|
||||
pub host: Option<TripleName>,
|
||||
|
||||
/// Container image to run the job in, using GitHub's builtin
|
||||
/// container support, see <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/running-jobs-in-a-container>
|
||||
///
|
||||
/// This doesn't allow mounting volumes, or anything, because we're only able
|
||||
/// to set the `container` key to something stringy
|
||||
///
|
||||
/// If not specified, `runner` has to be set.
|
||||
pub container: Option<StringLikeOr<ContainerImage, ContainerConfigInput>>,
|
||||
}
|
||||
|
||||
/// GitHub config that's common between different kinds of jobs (global, local)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct GithubRunnerConfig {
|
||||
/// GHA's `runs-on` key: Github Runner image to use, see <https://github.com/actions/runner-images>
|
||||
/// and <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/choosing-the-runner-for-a-job>
|
||||
///
|
||||
/// This is not necessarily a well-known runner, it could be something self-hosted, it
|
||||
/// could be from BuildJet, Namespace, etc.
|
||||
pub runner: GithubRunner,
|
||||
|
||||
/// Host triple of the runner (well-known, custom, or best guess).
|
||||
/// If the runner is one of GitHub's official runner images, the platform
|
||||
/// is hardcoded. If it's custom, then we have a `target_triple => runner` in the config
|
||||
pub host: TripleName,
|
||||
|
||||
/// Container image to run the job in, using GitHub's builtin
|
||||
/// container support, see <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/running-jobs-in-a-container>
|
||||
///
|
||||
/// This doesn't allow mounting volumes, or anything, because we're only able
|
||||
/// to set the `container` key to something stringy
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub container: Option<ContainerConfig>,
|
||||
}
|
||||
|
||||
impl GithubRunnerConfig {
|
||||
/// If the container runs through a container, that container might have a different
|
||||
/// architecture than the outer VM — this returns the container's triple if any,
|
||||
/// and falls back to the "machine"'s triple if not.
|
||||
pub fn real_triple_name(&self) -> &TripleNameRef {
|
||||
if let Some(container) = &self.container {
|
||||
&container.host
|
||||
} else {
|
||||
&self.host
|
||||
}
|
||||
}
|
||||
|
||||
/// cf. [`Self::GithubRunnerConfig`], but parsed
|
||||
pub fn real_triple(&self) -> Triple {
|
||||
self.real_triple_name().parse().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// GitHub config that's common between different kinds of jobs (global, local)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ContainerConfigInput {
|
||||
/// The container image to run, something like `ubuntu:20.04` or
|
||||
/// `quay.io/pypa/manylinux_2_28_x86_64`
|
||||
pub image: ContainerImage,
|
||||
|
||||
/// The host triple of the container, something like `x86_64-unknown-linux-gnu`
|
||||
/// or `aarch64-unknown-linux-musl` or whatever.
|
||||
pub host: Option<TripleName>,
|
||||
}
|
||||
|
||||
/// GitHub config that's common between different kinds of jobs (global, local)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct ContainerConfig {
|
||||
/// The container image to run, something like `ubuntu:20.04` or
|
||||
/// `quay.io/pypa/manylinux_2_28_x86_64`
|
||||
pub image: ContainerImage,
|
||||
|
||||
/// The host triple of the container, something like `x86_64-unknown-linux-gnu`
|
||||
/// or `aarch64-unknown-linux-musl` or whatever.
|
||||
pub host: TripleName,
|
||||
}
|
||||
|
||||
/// Used in `github/release.yml.j2` to template out "global" build jobs
|
||||
/// (plan, global assets, announce, etc)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct GithubGlobalJobConfig {
|
||||
/// Where to run this job?
|
||||
#[serde(flatten)]
|
||||
pub runner: GithubRunnerConfig,
|
||||
|
||||
/// Expression to execute to install dist
|
||||
pub install_dist: GhaRunStep,
|
||||
/// Expression to execute to install cargo-auditable
|
||||
pub install_cargo_auditable: GhaRunStep,
|
||||
|
||||
/// Arguments to pass to dist
|
||||
pub dist_args: String,
|
||||
|
||||
/// Expression to execute to install cargo-cyclonedx
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub install_cargo_cyclonedx: Option<GhaRunStep>,
|
||||
}
|
||||
|
||||
/// Used in `github/release.yml.j2` to template out "local" build jobs
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct GithubLocalJobConfig {
|
||||
/// Where to run this job?
|
||||
#[serde(flatten)]
|
||||
pub runner: GithubRunnerConfig,
|
||||
|
||||
/// Expression to execute to install dist
|
||||
pub install_dist: GhaRunStep,
|
||||
|
||||
/// Arguments to pass to dist
|
||||
pub dist_args: String,
|
||||
|
||||
/// Target triples to build for
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dist_args: Option<String>,
|
||||
pub targets: Option<Vec<TripleName>>,
|
||||
|
||||
/// Expression to execute to install cargo-auditable
|
||||
pub install_cargo_auditable: GhaRunStep,
|
||||
|
||||
/// Command to run to install dependencies
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub packages_install: Option<String>,
|
||||
/// what cache provider to use
|
||||
pub packages_install: Option<PackageInstallScript>,
|
||||
|
||||
/// What cache provider to use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_provider: Option<String>,
|
||||
}
|
||||
@ -475,7 +633,7 @@ pub struct Artifact {
|
||||
/// The target triple of the bundle
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
#[serde(default)]
|
||||
pub target_triples: Vec<TargetTriple>,
|
||||
pub target_triples: Vec<TripleName>,
|
||||
/// The location of the artifact on the local system
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(default)]
|
||||
@ -911,6 +1069,23 @@ pub enum PackageManager {
|
||||
Apt,
|
||||
}
|
||||
|
||||
declare_strongly_typed_string! {
|
||||
/// A homebrew package name, cf. <https://formulae.brew.sh/>
|
||||
pub struct HomebrewPackageName => &HomebrewPackageNameRef;
|
||||
|
||||
/// An APT package name, cf. <https://en.wikipedia.org/wiki/APT_(software)>
|
||||
pub struct AptPackageName => &AptPackageNameRef;
|
||||
|
||||
/// A chocolatey package name, cf. <https://community.chocolatey.org/packages>
|
||||
pub struct ChocolateyPackageName => &ChocolateyPackageNameRef;
|
||||
|
||||
/// A pip package name
|
||||
pub struct PipPackageName => &PipPackageNameRef;
|
||||
|
||||
/// A package version
|
||||
pub struct PackageVersion => &PackageVersionRef;
|
||||
}
|
||||
|
||||
/// Represents a dynamic library located somewhere on the system
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Library {
|
||||
@ -921,6 +1096,9 @@ pub struct Library {
|
||||
pub source: Option<String>,
|
||||
/// Which package manager provided this library
|
||||
pub package_manager: Option<PackageManager>,
|
||||
// FIXME: `HomebrewPackageName` and others are now strongly-typed, which makes having this
|
||||
// source/packagemanager thingy problematic. Maybe we could just have an enum, with Apt,
|
||||
// Homebrew, and Chocolatey variants? That would change the schema though.
|
||||
}
|
||||
|
||||
impl Linkage {
|
||||
@ -940,31 +1118,6 @@ impl Linkage {
|
||||
self.other.extend(other.iter().cloned());
|
||||
self.frameworks.extend(frameworks.iter().cloned());
|
||||
}
|
||||
|
||||
/// Returns a flat list of packages that come from the specific package manager
|
||||
pub fn packages_from(&self, package_manager: PackageManager) -> Vec<String> {
|
||||
let mut packages = vec![];
|
||||
packages.extend(
|
||||
self.system
|
||||
.iter()
|
||||
.filter(|l| l.package_manager == Some(package_manager))
|
||||
.filter_map(|l| l.source.clone()),
|
||||
);
|
||||
packages.extend(
|
||||
self.homebrew
|
||||
.iter()
|
||||
.filter(|l| l.package_manager == Some(package_manager))
|
||||
.filter_map(|l| l.source.clone()),
|
||||
);
|
||||
packages.extend(
|
||||
self.other
|
||||
.iter()
|
||||
.filter(|l| l.package_manager == Some(package_manager))
|
||||
.filter_map(|l| l.source.clone()),
|
||||
);
|
||||
|
||||
packages
|
||||
}
|
||||
}
|
||||
|
||||
impl Library {
|
||||
|
||||
@ -110,7 +110,7 @@
|
||||
/// ```rust,ignore
|
||||
/// declare_strongly_typed_string! {
|
||||
/// /// TargetTriple docs go here
|
||||
/// pub const TargetTriple => &TargetTripleRef;
|
||||
/// pub const TargetTriple => &TripleNameRef;
|
||||
///
|
||||
/// /// GithubRunner docs go here
|
||||
/// pub const GithubRunner => &GithubRunner;
|
||||
@ -133,14 +133,14 @@
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// If you're only reading from it, then maybe you can take a `&TargetTripleRef` instead:
|
||||
/// If you're only reading from it, then maybe you can take a `&TripleNameRef` instead:
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// fn is_target_triple_funny(target: &str) -> bool {
|
||||
/// target.contains("loong") // let's be honest: it's kinda funny
|
||||
/// }
|
||||
/// // 👇 becomes
|
||||
/// fn is_target_triple_funny(target: &TargetTripleRef) -> bool {
|
||||
/// fn is_target_triple_funny(target: &TripleNameRef) -> bool {
|
||||
/// target.as_str().contains("loong")
|
||||
/// }
|
||||
/// ```
|
||||
@ -173,8 +173,8 @@
|
||||
/// let target = TargetTriple::new("x86_64-unknown-linux-gnu");
|
||||
/// ```
|
||||
///
|
||||
/// What you're getting here is a `&'static TargetTripleRef` — no allocations
|
||||
/// involved, and if your functions take `&TargetTripleRef`, then you're already
|
||||
/// What you're getting here is a `&'static TripleNameRef` — no allocations
|
||||
/// involved, and if your functions take `&TripleNameRef`, then you're already
|
||||
/// all set.
|
||||
///
|
||||
/// ### Treating it as a string anyway
|
||||
@ -234,14 +234,14 @@
|
||||
/// This will not work:
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// fn i_take_a_slice(targets: &[TargetTripleRef]) { todo!(targets) }
|
||||
/// fn i_take_a_slice(targets: &[TripleNameRef]) { todo!(targets) }
|
||||
///
|
||||
/// let targets = vec![TargetTriple::new("x86_64-unknown-linux-gnu")];
|
||||
/// i_take_a_slice(&targets);
|
||||
/// ```
|
||||
///
|
||||
/// Because you have a `&Vec<TargetTriple>`, and `Deref` only takes you
|
||||
/// as far as `&[TargetTriple]`, but not `&[TargetTripleRef]`. If you
|
||||
/// as far as `&[TargetTriple]`, but not `&[TripleNameRef]`. If you
|
||||
/// encounter that case, it's probably fine to just take a `&[TargetTriple]`.
|
||||
///
|
||||
/// Note that you would have the same problem with `Vec<String>`: it would give
|
||||
@ -253,7 +253,7 @@
|
||||
/// This will not work:
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// fn match_on_target(target: &TargetTripleRef) => &str {
|
||||
/// fn match_on_target(target: &TripleNameRef) => &str {
|
||||
/// match target {
|
||||
/// TARGET_X64_WINDOWS => "what's up gamers",
|
||||
/// _ => "good morning",
|
||||
@ -261,7 +261,7 @@
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Even if `TARGET_X64_WINDOWS` is a `&'static TargetTripleRef` and
|
||||
/// Even if `TARGET_X64_WINDOWS` is a `&'static TripleNameRef` and
|
||||
/// a `const`. Doesn't matter. rustc says no. Maybe in the future.
|
||||
///
|
||||
/// For now, just stick it in a `HashMap`, or use an if-else chain or something. Sorry!
|
||||
|
||||
@ -671,6 +671,24 @@ snapshot_kind: text
|
||||
}
|
||||
}
|
||||
},
|
||||
"ContainerConfig": {
|
||||
"description": "GitHub config that's common between different kinds of jobs (global, local)",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"host",
|
||||
"image"
|
||||
],
|
||||
"properties": {
|
||||
"host": {
|
||||
"description": "The host triple of the container, something like `x86_64-unknown-linux-gnu` or `aarch64-unknown-linux-musl` or whatever.",
|
||||
"type": "string"
|
||||
},
|
||||
"image": {
|
||||
"description": "The container image to run, something like `ubuntu:20.04` or `quay.io/pypa/manylinux_2_28_x86_64`",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"EnvironmentVariables": {
|
||||
"description": "Release-specific environment variables",
|
||||
"type": "object",
|
||||
@ -815,40 +833,42 @@ snapshot_kind: text
|
||||
}
|
||||
}
|
||||
},
|
||||
"GithubMatrix": {
|
||||
"description": "Github CI Matrix",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"include": {
|
||||
"description": "define each task manually rather than doing cross-product stuff",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GithubMatrixEntry"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"GithubMatrixEntry": {
|
||||
"description": "Entry for a github matrix",
|
||||
"GithubLocalJobConfig": {
|
||||
"description": "Used in `github/release.yml.j2` to template out \"local\" build jobs",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"dist_args",
|
||||
"host",
|
||||
"install_cargo_auditable",
|
||||
"install_dist"
|
||||
"install_dist",
|
||||
"runner"
|
||||
],
|
||||
"properties": {
|
||||
"cache_provider": {
|
||||
"description": "what cache provider to use",
|
||||
"description": "What cache provider to use",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"container": {
|
||||
"description": "Container image to run the job in, using GitHub's builtin container support, see <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/running-jobs-in-a-container>\n\nThis doesn't allow mounting volumes, or anything, because we're only able to set the `container` key to something stringy",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ContainerConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dist_args": {
|
||||
"description": "Arguments to pass to dist",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
"type": "string"
|
||||
},
|
||||
"host": {
|
||||
"description": "Host triple of the runner (well-known, custom, or best guess). If the runner is one of GitHub's official runner images, the platform is hardcoded. If it's custom, then we have a `target_triple => runner` in the config",
|
||||
"type": "string"
|
||||
},
|
||||
"install_cargo_auditable": {
|
||||
"description": "Expression to execute to install cargo-auditable",
|
||||
@ -858,17 +878,6 @@ snapshot_kind: text
|
||||
}
|
||||
]
|
||||
},
|
||||
"install_cargo_cyclonedx": {
|
||||
"description": "Expression to execute to install cargo-cyclonedx",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/GhaRunStep"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"install_dist": {
|
||||
"description": "Expression to execute to install dist",
|
||||
"allOf": [
|
||||
@ -885,14 +894,11 @@ snapshot_kind: text
|
||||
]
|
||||
},
|
||||
"runner": {
|
||||
"description": "Github Runner to user",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
"description": "GHA's `runs-on` key: Github Runner image to use, see <https://github.com/actions/runner-images> and <https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/choosing-the-runner-for-a-job>\n\nThis is not necessarily a well-known runner, it could be something self-hosted, it could be from BuildJet, Namespace, etc.",
|
||||
"type": "string"
|
||||
},
|
||||
"targets": {
|
||||
"description": "Targets to build for",
|
||||
"description": "Target triples to build for",
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
@ -903,6 +909,19 @@ snapshot_kind: text
|
||||
}
|
||||
}
|
||||
},
|
||||
"GithubMatrix": {
|
||||
"description": "Github CI Matrix",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"include": {
|
||||
"description": "define each task manually rather than doing cross-product stuff",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GithubLocalJobConfig"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"GlibcVersion": {
|
||||
"description": "Minimum glibc version required to run software",
|
||||
"type": "object",
|
||||
|
||||
@ -90,7 +90,7 @@ use std::fmt::Display;
|
||||
|
||||
use axoproject::PackageIdx;
|
||||
use axotag::{parse_tag, Package, PartialAnnouncementTag, ReleaseType};
|
||||
use cargo_dist_schema::{DistManifest, GithubHosting, TargetTriple, TargetTripleRef};
|
||||
use cargo_dist_schema::{DistManifest, GithubHosting, TripleName, TripleNameRef};
|
||||
use itertools::Itertools;
|
||||
use semver::Version;
|
||||
use tracing::info;
|
||||
@ -977,7 +977,7 @@ pub fn announcement_github(manifest: &mut DistManifest) {
|
||||
}
|
||||
|
||||
/// Create a key for Properly sorting a list of target triples
|
||||
fn sortable_triples(triples: &[TargetTriple]) -> Vec<Vec<String>> {
|
||||
fn sortable_triples(triples: &[TripleName]) -> Vec<Vec<String>> {
|
||||
// Make each triple sortable, and then sort the list of triples by those
|
||||
// (usually there's only one triple but DETERMINISM)
|
||||
let mut output: Vec<Vec<String>> = triples.iter().map(|t| sortable_triple(t)).collect();
|
||||
@ -986,7 +986,7 @@ fn sortable_triples(triples: &[TargetTriple]) -> Vec<Vec<String>> {
|
||||
}
|
||||
|
||||
/// Create a key for Properly sorting target triples
|
||||
fn sortable_triple(triple: &TargetTripleRef) -> Vec<String> {
|
||||
fn sortable_triple(triple: &TripleNameRef) -> Vec<String> {
|
||||
// We want to sort lexically by: os, abi, arch
|
||||
// We are given arch, vendor, os, abi
|
||||
//
|
||||
@ -1018,7 +1018,7 @@ fn sortable_triple(triple: &TargetTripleRef) -> Vec<String> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use cargo_dist_schema::TargetTripleRef;
|
||||
use cargo_dist_schema::TripleNameRef;
|
||||
|
||||
use super::sortable_triple;
|
||||
#[test]
|
||||
@ -1045,7 +1045,7 @@ mod tests {
|
||||
"x86_64-unknown-linux-gnu.2.31",
|
||||
"x86_64-unknown-linux-musl-static",
|
||||
];
|
||||
targets.sort_by_cached_key(|t| sortable_triple(TargetTripleRef::from_str(t)));
|
||||
targets.sort_by_cached_key(|t| sortable_triple(TripleNameRef::from_str(t)));
|
||||
assert_eq!(
|
||||
targets,
|
||||
vec![
|
||||
|
||||
@ -8,21 +8,26 @@ use axoasset::{LocalAsset, SourceFile};
|
||||
use axoprocess::Cmd;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
use cargo_dist_schema::{
|
||||
GhaRunStep, GithubMatrix, GithubMatrixEntry, GithubRunner, GithubRunnerRef, TargetTriple,
|
||||
TargetTripleRef,
|
||||
target_lexicon::{self, OperatingSystem, Triple},
|
||||
AptPackageName, ChocolateyPackageName, GhaRunStep, GithubGlobalJobConfig, GithubLocalJobConfig,
|
||||
GithubMatrix, GithubRunnerConfig, GithubRunnerRef, GithubRunners, HomebrewPackageName,
|
||||
PackageInstallScript, PackageVersion, PipPackageName, TripleNameRef,
|
||||
};
|
||||
use itertools::Itertools;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::{
|
||||
backend::{diff_files, templates::TEMPLATE_CI_GITHUB},
|
||||
build_wrapper_for_cross,
|
||||
config::{
|
||||
v1::{ci::github::GithubCiConfig, publishers::PublisherConfig},
|
||||
DependencyKind, GithubPermission, GithubPermissionMap, GithubReleasePhase, HostingStyle,
|
||||
JinjaGithubRepoPair, JobStyle, ProductionMode, PublishStyle, SystemDependencies,
|
||||
},
|
||||
errors::DistResult,
|
||||
DistError, DistGraph, SortedMap, SortedSet,
|
||||
platform::github_runners::target_for_github_runner_or_default,
|
||||
CargoBuildWrapper, DistError, DistGraph, SortedMap, SortedSet,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@ -65,7 +70,7 @@ pub struct GithubCiInfo {
|
||||
/// What kind of job to run on pull request
|
||||
pub pr_run_mode: cargo_dist_schema::PrRunMode,
|
||||
/// global task
|
||||
pub global_task: GithubMatrixEntry,
|
||||
pub global_task: GithubGlobalJobConfig,
|
||||
/// homebrew tap
|
||||
pub tap: Option<String>,
|
||||
/// plan jobs
|
||||
@ -222,7 +227,7 @@ impl GithubCiInfo {
|
||||
let need_cargo_cyclonedx = dist.config.builds.cargo.cargo_cyclonedx;
|
||||
|
||||
// Figure out what builds we need to do
|
||||
let mut local_targets: SortedSet<&TargetTripleRef> = SortedSet::new();
|
||||
let mut local_targets: SortedSet<&TripleNameRef> = SortedSet::new();
|
||||
for release in &dist.releases {
|
||||
for target in &release.targets {
|
||||
local_targets.insert(target);
|
||||
@ -258,17 +263,13 @@ impl GithubCiInfo {
|
||||
let global_runner = ci_config
|
||||
.runners
|
||||
.get("global")
|
||||
.map(|s| s.as_ref())
|
||||
.unwrap_or(GITHUB_LINUX_RUNNER);
|
||||
let global_task = GithubMatrixEntry {
|
||||
targets: None,
|
||||
cache_provider: cache_provider_for_runner(global_runner),
|
||||
runner: Some(global_runner.to_owned()),
|
||||
dist_args: Some("--artifacts=global".into()),
|
||||
.cloned()
|
||||
.unwrap_or_else(default_global_runner_config);
|
||||
let global_task = GithubGlobalJobConfig {
|
||||
runner: global_runner.to_owned(),
|
||||
dist_args: "--artifacts=global".into(),
|
||||
install_dist: dist_install_strategy.dash(),
|
||||
install_cargo_auditable: cargo_auditable_install_strategy.dash(),
|
||||
install_cargo_cyclonedx: Some(cargo_cyclonedx_install_strategy.dash()),
|
||||
packages_install: None,
|
||||
};
|
||||
|
||||
let tap = dist.global_homebrew_tap.clone();
|
||||
@ -322,22 +323,24 @@ impl GithubCiInfo {
|
||||
};
|
||||
for (runner, targets) in local_runs {
|
||||
use std::fmt::Write;
|
||||
let install_dist = dist_install_strategy.for_targets(&targets);
|
||||
let install_cargo_auditable = cargo_auditable_install_strategy.for_targets(&targets);
|
||||
let real_triple = runner.real_triple();
|
||||
let install_dist = dist_install_strategy.for_triple(&real_triple);
|
||||
let install_cargo_auditable =
|
||||
cargo_auditable_install_strategy.for_triple(&runner.real_triple());
|
||||
|
||||
let mut dist_args = String::from("--artifacts=local");
|
||||
for target in &targets {
|
||||
write!(dist_args, " --target={target}").unwrap();
|
||||
}
|
||||
tasks.push(GithubMatrixEntry {
|
||||
let packages_install = system_deps_install_script(&runner, &targets, &dependencies)?;
|
||||
tasks.push(GithubLocalJobConfig {
|
||||
targets: Some(targets.iter().copied().map(|s| s.to_owned()).collect()),
|
||||
cache_provider: cache_provider_for_runner(&runner),
|
||||
runner: Some(runner),
|
||||
dist_args: Some(dist_args),
|
||||
runner,
|
||||
dist_args,
|
||||
install_dist: install_dist.to_owned(),
|
||||
install_cargo_auditable: install_cargo_auditable.to_owned(),
|
||||
install_cargo_cyclonedx: None, // Not used by local builds.
|
||||
packages_install: package_install_for_targets(&targets, &dependencies),
|
||||
packages_install,
|
||||
});
|
||||
}
|
||||
|
||||
@ -554,8 +557,8 @@ fn build_jobs(
|
||||
/// Get the best `cache-provider` key to use for <https://github.com/Swatinem/rust-cache>.
|
||||
///
|
||||
/// In the future we might make "None" here be a way to say "disable the cache".
|
||||
fn cache_provider_for_runner(runner: &GithubRunnerRef) -> Option<String> {
|
||||
if runner.is_buildjet() {
|
||||
fn cache_provider_for_runner(rc: &GithubRunnerConfig) -> Option<String> {
|
||||
if rc.runner.is_buildjet() {
|
||||
Some("buildjet".into())
|
||||
} else {
|
||||
Some("github".into())
|
||||
@ -572,20 +575,23 @@ fn cache_provider_for_runner(runner: &GithubRunnerRef) -> Option<String> {
|
||||
/// macos builds. It also makes it impossible to have one macos build fail and the other
|
||||
/// succeed (uploading itself to the draft release).
|
||||
///
|
||||
/// In priniciple it does remove some duplicated setup work, so this is ostensibly "cheaper".
|
||||
/// In principle it does remove some duplicated setup work, so this is ostensibly "cheaper".
|
||||
fn distribute_targets_to_runners_merged<'a>(
|
||||
targets: SortedSet<&'a TargetTripleRef>,
|
||||
custom_runners: &BTreeMap<TargetTriple, GithubRunner>,
|
||||
) -> std::vec::IntoIter<(GithubRunner, Vec<&'a TargetTripleRef>)> {
|
||||
let mut groups = SortedMap::<GithubRunner, Vec<&TargetTripleRef>>::new();
|
||||
targets: SortedSet<&'a TripleNameRef>,
|
||||
custom_runners: &GithubRunners,
|
||||
) -> std::vec::IntoIter<(GithubRunnerConfig, Vec<&'a TripleNameRef>)> {
|
||||
let mut groups = SortedMap::<GithubRunnerConfig, Vec<&TripleNameRef>>::new();
|
||||
for target in targets {
|
||||
let runner = github_runner_for_target(target, custom_runners);
|
||||
let runner = runner.unwrap_or_else(|| {
|
||||
let default = GITHUB_LINUX_RUNNER;
|
||||
warn!("not sure which github runner should be used for {target}, assuming {default}");
|
||||
default.to_owned()
|
||||
let runner_conf = github_runner_for_target(target, custom_runners);
|
||||
let runner_conf = runner_conf.unwrap_or_else(|| {
|
||||
let fallback = default_global_runner_config();
|
||||
warn!(
|
||||
"not sure which github runner should be used for {target}, assuming {}",
|
||||
fallback.runner
|
||||
);
|
||||
fallback.to_owned()
|
||||
});
|
||||
groups.entry(runner).or_default().push(target);
|
||||
groups.entry(runner_conf).or_default().push(target);
|
||||
}
|
||||
// This extra into_iter+collect is needed to make this have the same
|
||||
// return type as distribute_targets_to_runners_split
|
||||
@ -595,61 +601,66 @@ fn distribute_targets_to_runners_merged<'a>(
|
||||
/// Given a set of targets we want to build local artifacts for, map them to Github Runners
|
||||
/// while preferring each target gets its own runner for latency and fault-isolation.
|
||||
fn distribute_targets_to_runners_split<'a>(
|
||||
targets: SortedSet<&'a TargetTripleRef>,
|
||||
custom_runners: &BTreeMap<TargetTriple, GithubRunner>,
|
||||
) -> std::vec::IntoIter<(GithubRunner, Vec<&'a TargetTripleRef>)> {
|
||||
targets: SortedSet<&'a TripleNameRef>,
|
||||
custom_runners: &GithubRunners,
|
||||
) -> std::vec::IntoIter<(GithubRunnerConfig, Vec<&'a TripleNameRef>)> {
|
||||
let mut groups = vec![];
|
||||
for target in targets {
|
||||
let runner = github_runner_for_target(target, custom_runners);
|
||||
let runner = runner.unwrap_or_else(|| {
|
||||
let default = GITHUB_LINUX_RUNNER;
|
||||
warn!("not sure which github runner should be used for {target}, assuming {default}");
|
||||
default.to_owned()
|
||||
let fallback = default_global_runner_config();
|
||||
warn!(
|
||||
"not sure which github runner should be used for {target}, assuming {}",
|
||||
fallback.runner
|
||||
);
|
||||
fallback.to_owned()
|
||||
});
|
||||
groups.push((runner, vec![target]));
|
||||
}
|
||||
groups.into_iter()
|
||||
}
|
||||
|
||||
/// The Github Runner to use for Linux
|
||||
const GITHUB_LINUX_RUNNER: &GithubRunnerRef = GithubRunnerRef::from_str("ubuntu-20.04");
|
||||
/// The Github Runner to use for Intel macos
|
||||
const GITHUB_MACOS_INTEL_RUNNER: &GithubRunnerRef = GithubRunnerRef::from_str("macos-13");
|
||||
/// The Github Runner to use for Apple Silicon macos
|
||||
const GITHUB_MACOS_ARM64_RUNNER: &GithubRunnerRef = GithubRunnerRef::from_str("macos-13");
|
||||
/// The Github Runner to use for windows
|
||||
const GITHUB_WINDOWS_RUNNER: &GithubRunnerRef = GithubRunnerRef::from_str("windows-2019");
|
||||
/// Generates a [`GithubRunnerConfig`] from a given github runner name
|
||||
pub fn runner_to_config(runner: &GithubRunnerRef) -> GithubRunnerConfig {
|
||||
GithubRunnerConfig {
|
||||
runner: runner.to_owned(),
|
||||
host: target_for_github_runner_or_default(runner).to_owned(),
|
||||
container: None,
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_LINUX_RUNNER: &GithubRunnerRef = GithubRunnerRef::from_str("ubuntu-20.04");
|
||||
|
||||
fn default_global_runner_config() -> GithubRunnerConfig {
|
||||
runner_to_config(DEFAULT_LINUX_RUNNER)
|
||||
}
|
||||
|
||||
/// Get the appropriate Github Runner for building a target
|
||||
fn github_runner_for_target(
|
||||
target: &TargetTripleRef,
|
||||
custom_runners: &BTreeMap<TargetTriple, GithubRunner>,
|
||||
) -> Option<GithubRunner> {
|
||||
target: &TripleNameRef,
|
||||
custom_runners: &GithubRunners,
|
||||
) -> Option<GithubRunnerConfig> {
|
||||
if let Some(runner) = custom_runners.get(target) {
|
||||
return Some(runner.to_owned());
|
||||
return Some(runner.clone());
|
||||
}
|
||||
|
||||
let target_triple: Triple = target.parse().unwrap();
|
||||
|
||||
// We want to default to older runners to minimize the places
|
||||
// where random system dependencies can creep in and be very
|
||||
// recent. This helps with portability!
|
||||
if target.is_linux() {
|
||||
Some(GITHUB_LINUX_RUNNER.to_owned())
|
||||
} else if target.is_apple() && target.is_x86_64() {
|
||||
Some(GITHUB_MACOS_INTEL_RUNNER.to_owned())
|
||||
} else if target.is_apple() && target.is_aarch64() {
|
||||
Some(GITHUB_MACOS_ARM64_RUNNER.to_owned())
|
||||
} else if target.is_windows() {
|
||||
Some(GITHUB_WINDOWS_RUNNER.to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
Some(match target_triple.operating_system {
|
||||
OperatingSystem::Linux => runner_to_config(GithubRunnerRef::from_str("ubuntu-20.04")),
|
||||
OperatingSystem::Darwin => runner_to_config(GithubRunnerRef::from_str("macos-13")),
|
||||
OperatingSystem::Windows => runner_to_config(GithubRunnerRef::from_str("windows-2019")),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn brewfile_from(packages: &[String]) -> String {
|
||||
let brewfile_lines: Vec<String> = packages
|
||||
.iter()
|
||||
fn brewfile_from<'a>(packages: impl Iterator<Item = &'a HomebrewPackageName>) -> String {
|
||||
packages
|
||||
.map(|p| {
|
||||
let lower = p.to_ascii_lowercase();
|
||||
let lower = p.as_str().to_ascii_lowercase();
|
||||
// Although `brew install` can take either a formula or a cask,
|
||||
// Brewfiles require you to use the `cask` verb for casks and `brew`
|
||||
// for formulas.
|
||||
@ -659,12 +670,10 @@ fn brewfile_from(packages: &[String]) -> String {
|
||||
format!(r#"brew "{p}""#).to_owned()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
brewfile_lines.join("\n")
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn brew_bundle_command(packages: &[String]) -> String {
|
||||
fn brew_bundle_command<'a>(packages: impl Iterator<Item = &'a HomebrewPackageName>) -> String {
|
||||
format!(
|
||||
r#"cat << EOF >Brewfile
|
||||
{}
|
||||
@ -675,92 +684,156 @@ brew bundle install"#,
|
||||
)
|
||||
}
|
||||
|
||||
fn package_install_for_targets(
|
||||
targets: &[&TargetTripleRef],
|
||||
fn system_deps_install_script(
|
||||
rc: &GithubRunnerConfig,
|
||||
targets: &[&TripleNameRef],
|
||||
packages: &SystemDependencies,
|
||||
) -> Option<String> {
|
||||
// FIXME?: handle mixed-OS targets
|
||||
for target in targets {
|
||||
match target.as_str() {
|
||||
"i686-apple-darwin" | "x86_64-apple-darwin" | "aarch64-apple-darwin" => {
|
||||
let packages: Vec<String> = packages
|
||||
.homebrew
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(|(_, package)| package.0.wanted_for_target(target))
|
||||
.filter(|(_, package)| package.0.stage_wanted(&DependencyKind::Build))
|
||||
.map(|(name, _)| name)
|
||||
.collect();
|
||||
) -> DistResult<Option<PackageInstallScript>> {
|
||||
let mut brew_packages: SortedSet<HomebrewPackageName> = Default::default();
|
||||
let mut apt_packages: SortedSet<(AptPackageName, Option<PackageVersion>)> = Default::default();
|
||||
let mut chocolatey_packages: SortedSet<(ChocolateyPackageName, Option<PackageVersion>)> =
|
||||
Default::default();
|
||||
|
||||
if packages.is_empty() {
|
||||
return None;
|
||||
let host = rc.real_triple();
|
||||
match host.operating_system {
|
||||
OperatingSystem::Darwin => {
|
||||
for (name, pkg) in &packages.homebrew {
|
||||
if !pkg.0.stage_wanted(&DependencyKind::Build) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return Some(brew_bundle_command(&packages));
|
||||
if !targets.iter().any(|target| pkg.0.wanted_for_target(target)) {
|
||||
continue;
|
||||
}
|
||||
brew_packages.insert(name.clone());
|
||||
}
|
||||
}
|
||||
OperatingSystem::Linux => {
|
||||
for (name, pkg) in &packages.apt {
|
||||
if !pkg.0.stage_wanted(&DependencyKind::Build) {
|
||||
continue;
|
||||
}
|
||||
if !targets.iter().any(|target| pkg.0.wanted_for_target(target)) {
|
||||
continue;
|
||||
}
|
||||
apt_packages.insert((name.clone(), pkg.0.version.clone()));
|
||||
}
|
||||
"i686-unknown-linux-gnu"
|
||||
| "x86_64-unknown-linux-gnu"
|
||||
| "aarch64-unknown-linux-gnu"
|
||||
| "i686-unknown-linux-musl"
|
||||
| "x86_64-unknown-linux-musl"
|
||||
| "aarch64-unknown-linux-musl" => {
|
||||
let mut packages: Vec<String> = packages
|
||||
.apt
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(|(_, package)| package.0.wanted_for_target(target))
|
||||
.filter(|(_, package)| package.0.stage_wanted(&DependencyKind::Build))
|
||||
.map(|(name, spec)| {
|
||||
if let Some(version) = spec.0.version {
|
||||
format!("{name}={version}")
|
||||
} else {
|
||||
name
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let has_musl_target = targets.iter().any(|target| {
|
||||
target.parse().unwrap().environment == target_lexicon::Environment::Musl
|
||||
});
|
||||
if has_musl_target {
|
||||
// musl builds may require musl-tools to build;
|
||||
// necessary for more complex software
|
||||
if target.is_linux_musl() {
|
||||
packages.push("musl-tools".to_owned());
|
||||
}
|
||||
|
||||
if packages.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let apts = packages.join(" ");
|
||||
return Some(
|
||||
format!("sudo apt-get update && sudo apt-get install {apts}").to_owned(),
|
||||
);
|
||||
apt_packages.insert((AptPackageName::new("musl-tools".to_owned()), None));
|
||||
}
|
||||
"i686-pc-windows-msvc" | "x86_64-pc-windows-msvc" | "aarch64-pc-windows-msvc" => {
|
||||
let commands: Vec<String> = packages
|
||||
.chocolatey
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(|(_, package)| package.0.wanted_for_target(target))
|
||||
.filter(|(_, package)| package.0.stage_wanted(&DependencyKind::Build))
|
||||
.map(|(name, package)| {
|
||||
if let Some(version) = package.0.version {
|
||||
format!("choco install {name} --version={version}")
|
||||
} else {
|
||||
format!("choco install {name}")
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if commands.is_empty() {
|
||||
return None;
|
||||
}
|
||||
OperatingSystem::Windows => {
|
||||
for (name, pkg) in &packages.chocolatey {
|
||||
if !pkg.0.stage_wanted(&DependencyKind::Build) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return Some(commands.join("\n"));
|
||||
if !targets.iter().any(|target| pkg.0.wanted_for_target(target)) {
|
||||
continue;
|
||||
}
|
||||
chocolatey_packages.insert((name.clone(), pkg.0.version.clone()));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
_ => {
|
||||
panic!(
|
||||
"unsupported host operating system: {:?}",
|
||||
host.operating_system
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
let mut lines = vec![];
|
||||
if !brew_packages.is_empty() {
|
||||
lines.push(brew_bundle_command(brew_packages.iter()))
|
||||
}
|
||||
|
||||
if !apt_packages.is_empty() {
|
||||
lines.push("sudo apt-get update".to_owned());
|
||||
let args = apt_packages
|
||||
.iter()
|
||||
.map(|(pkg, version)| {
|
||||
if let Some(v) = version {
|
||||
format!("{pkg}={v}")
|
||||
} else {
|
||||