Skip to content

Commit b0cfb6f

Browse files
threexcthreexc
committed
workflows: hf-xet: add build, test workflows
Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com>
1 parent 5b50e76 commit b0cfb6f

2 files changed

Lines changed: 227 additions & 0 deletions

File tree

‎.github/workflows/build-hf-xet.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# SPDX-FileCopyrightText: 2026 The RISE Project
2+
# SPDX-License-Identifier: MIT
3+
4+
name: Build hf-xet wheels (riscv64)
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
version:
10+
description: 'hf-xet version to build (git tag without leading v, e.g. 1.6.0)'
11+
required: true
12+
default: '1.6.0'
13+
pull_request:
14+
paths:
15+
- '.github/workflows/build-hf-xet.yml'
16+
- '.github/workflows/test-hf-xet.yml'
17+
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ inputs.version || '1.6.0' }}-${{ github.head_ref || github.run_id }}
20+
cancel-in-progress: true
21+
22+
permissions:
23+
contents: read # to fetch code (actions/checkout)
24+
25+
env:
26+
HFXET_VERSION: ${{ inputs.version || '1.6.0' }}
27+
28+
jobs:
29+
# The 3.14 wheel is abi3 compatible, so a separate build is only needed for
30+
# 3.14t (free-threaded CPython has no stable ABI to target yet).
31+
build_wheels:
32+
name: Build hf-xet ${{ inputs.version || '1.6.0' }} ${{ matrix.python-version }}-riscv64
33+
runs-on: ubuntu-24.04-riscv
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
platform:
38+
- target: riscv64gc-unknown-linux-gnu
39+
arch: riscv64
40+
manylinux: 2_39
41+
python-version:
42+
- '3.14'
43+
- '3.14t'
44+
45+
steps:
46+
- name: Checkout hf-xet v${{ env.HFXET_VERSION }}
47+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
48+
with:
49+
repository: huggingface/xet-core
50+
ref: v${{ env.HFXET_VERSION }}
51+
persist-credentials: false
52+
53+
- name: Install Python
54+
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
55+
with:
56+
python-version: ${{ matrix.python-version }}
57+
activate-environment: true
58+
enable-cache: false
59+
60+
- name: Install wheel tooling
61+
run: |
62+
uv pip install --upgrade pip
63+
pip install wheel
64+
65+
- uses: ./.github/actions/set-build-profile
66+
with:
67+
tag: v${{ env.HFXET_VERSION }}
68+
69+
- name: Build wheels
70+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
71+
with:
72+
target: ${{ matrix.platform.target }}
73+
args: --profile ${{ env.BUILD_PROFILE }} -i ${{ matrix.python-version }} --out dist
74+
sccache: 'true'
75+
manylinux: ${{ matrix.platform.manylinux }}
76+
working-directory: hf_xet
77+
before-script-linux: |
78+
if command -v apt-get &> /dev/null; then
79+
apt-get update && apt-get install libatomic-ops-dev -y
80+
elif command -v yum &> /dev/null; then
81+
yum install libatomic_ops-devel perl-IPC-Cmd -y
82+
else
83+
echo "Neither apt-get nor yum is installed. Please install a package manager."
84+
exit 1
85+
fi
86+
git config --global --add safe.directory "*"
87+
88+
- name: Strip debug symbols into a separate file
89+
if: env.IS_RELEASE == 'true'
90+
run: |
91+
mkdir hf_xet/dbg
92+
mkdir dist
93+
pushd dist
94+
cp ../hf_xet/dist/* .
95+
LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1)
96+
WHEEL_NAME=$(basename $LATEST_WHEEL .whl)
97+
WHEEL_VERSION=$(echo $LATEST_WHEEL | cut -d '-' -f 2)
98+
SYMBOL_FILE=${WHEEL_NAME}.so.dbg
99+
wheel unpack $LATEST_WHEEL
100+
rm *.whl
101+
objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE}
102+
objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
103+
objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
104+
mv hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} ../hf_xet/dbg/${SYMBOL_FILE}
105+
wheel pack hf_xet-${WHEEL_VERSION}
106+
rm -rf hf_xet-${WHEEL_VERSION}
107+
108+
- name: Copy unstripped wheels to upload directory
109+
if: env.IS_RELEASE != 'true'
110+
run: |
111+
mkdir dist
112+
cp hf_xet/dist/* dist/
113+
114+
- name: Upload debug symbols
115+
if: env.IS_RELEASE == 'true'
116+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
117+
with:
118+
name: hf-xet-${{ env.HFXET_VERSION }}-${{ matrix.python-version }}-riscv64-dbg
119+
path: hf_xet/dbg
120+
121+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
122+
with:
123+
name: hf-xet-${{ env.HFXET_VERSION }}-${{ matrix.python-version }}-riscv64
124+
path: dist/*.whl
125+
if-no-files-found: error
126+
127+
publish:
128+
name: Publish hf-xet ${{ inputs.version || '1.6.0' }} to GitLab
129+
needs: [build_wheels]
130+
runs-on: ubuntu-latest
131+
permissions:
132+
contents: write
133+
pull-requests: write
134+
135+
steps:
136+
- name: Publish wheels and open docs PR
137+
uses: riseproject-dev/python-wheels/actions/publish-wheels@main
138+
with:
139+
artifact-pattern: hf-xet-${{ env.HFXET_VERSION }}-*-riscv64
140+
gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }}
141+
gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }}
142+
gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }}
143+
gh-token: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/test-hf-xet.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# SPDX-FileCopyrightText: 2026 The RISE Project
2+
# SPDX-License-Identifier: MIT
3+
4+
name: Test hf-xet (riscv64)
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
version:
10+
description: 'hf-xet version to test (git tag without leading v, e.g. 1.6.0)'
11+
required: true
12+
default: '1.6.0'
13+
pull_request:
14+
paths:
15+
- '.github/workflows/test-hf-xet.yml'
16+
17+
concurrency:
18+
group: ${{ github.workflow }}-${{ inputs.version || '1.6.0' }}-${{ github.head_ref || github.run_id }}
19+
cancel-in-progress: true
20+
21+
permissions:
22+
contents: read
23+
24+
env:
25+
HFXET_VERSION: ${{ inputs.version || '1.6.0' }}
26+
27+
jobs:
28+
# Mirrors ci.yml's build_and_test-linux job: build the extension in place
29+
# with `maturin develop` and run hf_xet's own pytest suite against it.
30+
# hf-xet-tests.yml (huggingface_hub's xet integration tests) and the
31+
# workspace-wide `cargo test` in ci.yml are dropped - the former always
32+
# installs the released PyPI wheel rather than what's built here, and the
33+
# latter exercises git_xet/xet_client code outside the Python package.
34+
test:
35+
permissions:
36+
contents: read
37+
name: "Test hf-xet ${{ inputs.version || '1.6.0' }} — Python 3.12 on riscv64"
38+
runs-on: ubuntu-24.04-riscv
39+
40+
steps:
41+
- name: Checkout hf-xet v${{ env.HFXET_VERSION }}
42+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
43+
with:
44+
repository: huggingface/xet-core
45+
ref: v${{ env.HFXET_VERSION }}
46+
persist-credentials: false
47+
48+
- name: Install Python
49+
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
50+
with:
51+
python-version: '3.12'
52+
activate-environment: true
53+
enable-cache: false
54+
55+
- name: Install libpython for pyo3 linking
56+
run: |
57+
set -eux
58+
py_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
59+
sudo apt-get update
60+
if ! apt-cache show "libpython${py_ver}-dev" >/dev/null 2>&1; then
61+
sudo apt-get install -y software-properties-common
62+
sudo add-apt-repository -y ppa:deadsnakes/ppa
63+
sudo apt-get update
64+
fi
65+
sudo apt-get install -y "libpython${py_ver}-dev"
66+
67+
- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master
68+
with:
69+
toolchain: 1.94.1
70+
71+
- name: Build and Test hf_xet
72+
run: cd hf_xet && cargo test --verbose --no-fail-fast
73+
74+
- name: Build wheel
75+
uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
76+
with:
77+
command: develop
78+
sccache: 'true'
79+
working-directory: hf_xet
80+
81+
- name: Python integration tests (hf_xet)
82+
run: |
83+
pip install pytest
84+
pytest hf_xet/tests/ -v

0 commit comments

Comments
 (0)