diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..781e4e043 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,38 @@ +name: Build Distribution Files + +on: + push: + branches: [ fourdiff-pypi ] + pull_request: + branches: [ fourdiff-pypi ] + +jobs: + build: + runs-on: ubuntu-latest + container: + # Latest right now, based on ubuntu 24.04 + #image: condaforge/miniforge3:26.1.1-3 + image: ubuntu:24.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install ubuntu packages + run: | + apt-get update && build-essential pkg-config python + + # - name: install upterm requirements + # run: apt-get install -y --no-install-recommends tmux openssh-client + # - name: upterm ssh debug + # uses: owenthereal/action-upterm@v1 + + - name: Build the files + run: bundler/bundler.py --workdir work + + - name: Upload build artifacts + uses: actions/upload-artifact@v6 + with: + name: built-files + path: dist/ + retention-days: 7 diff --git a/.github/workflows/debug-ssh.yml b/.github/workflows/debug-ssh.yml index 73c37e710..a3710a699 100644 --- a/.github/workflows/debug-ssh.yml +++ b/.github/workflows/debug-ssh.yml @@ -7,7 +7,7 @@ on: description: 'Enable debug SSH?' required: false default: 'false' - pull_request: + #pull_request: jobs: build: diff --git a/bundler/bundler.py b/bundler/bundler.py index 84ed20266..307169278 100755 --- a/bundler/bundler.py +++ b/bundler/bundler.py @@ -7,7 +7,7 @@ import re import sys from pathlib import Path -from shutil import copy, copytree +from shutil import copy, copytree, which from subprocess import check_call from tempfile import mkdtemp, mkstemp @@ -323,11 +323,18 @@ def main(): parser = ArgumentParser(description="build and bundle") parser.add_argument("--workdir", type=Path, help="Path for work. If not given, will be chosen in /tmp") - parser.add_argument("conda", type=Path, help="path to conda executable") + parser.add_argument("--conda", type=Path, help="Path to conda executable. If not given, will be taken from PATH") args = parser.parse_args() workdir = args.workdir or Path(mkdtemp(prefix="meld-bundler-workdir-")) - build_all(args.conda, workdir) + if args.conda: + conda = args.conda + else: + conda0 = which('conda') + if conda0 is None: + raise RuntimeError("Couldn't find conda") + conda = Path(conda0) + build_all(conda, workdir) if __name__ == "__main__": diff --git a/bundler/dedup_solibs.py b/bundler/dedup_solibs.py new file mode 100644 index 000000000..c1e87e2d3 --- /dev/null +++ b/bundler/dedup_solibs.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import os +import re +from pathlib import Path +from subprocess import check_call +from tempfile import mkstemp + + +def get_solib_groups(env: Path) -> dict[str, list[str]]: + libdir = env / 'lib' + solib_groups: dict[str, list[str]] = {} + for fn in libdir.iterdir(): + name = fn.name + if '.so' not in name: + continue + base = name[:name.index('.so') + 3] + if base == 'libgcc_s.so': + # This is special, libgcc_s.so and libgcc_s.so.1 are different + continue + solib_groups.setdefault(base, []).append(name) + for names in solib_groups.values(): + # Sort fns so the first one is the shortest one, which should be copied + names.sort(key=lambda name: len(name)) + fns = [libdir.joinpath(name) for name in names] + # In some cases there are equal files which are not symlinks. So just read all of them. + bs = [fn.read_bytes() for fn in fns] + for fn, b in zip(fns, bs): + if b != bs[0]: + raise RuntimeError(f"Expecting {fns[0]} and {fn} to have the same content") + return solib_groups + + +def fix_solib_names_in_bytes(solib_groups: dict[str, list[str]], solib_re: re.Pattern[bytes], matches: list[re.Match[bytes]], b0: bytes, is_binary: bool) -> bytes: + if is_binary: + terminators = [0] + else: + terminators = [ord(','), ord('"')] + b = bytes() + last_pos = 0 + ignore_matches_before = 0 + for match in matches: + if match.start() < ignore_matches_before: + continue + short = match.group(0) + names = solib_groups[short.decode('ascii')] + for end_pos in range(match.start(), len(b0)): + if b0[end_pos] in terminators: + break + else: + # We accept the end of string as a terminator, for the recursive call. + end_pos = len(b0) + segment = b0[match.start():end_pos] + name = segment.decode('ascii') + if name not in names: + if name == 'libGLX.so.1': + # Probably refers to the OS + continue + if is_binary: + # We can have a string like "libharfbuzz-gobject.so.0,libharfbuzz.so.0" in a binary typelib file. + # So we call ourselves with is_binary=False, to try to replace it with a shorter string. + matches2 = list(solib_re.finditer(segment)) + short = fix_solib_names_in_bytes(solib_groups, solib_re, matches2, segment, is_binary=False) + ignore_matches_before = end_pos + else: + raise RuntimeError(f"Found string {name} but it's not in {names}") + + b += b0[last_pos:match.start()] + b += short + if is_binary: + b += b'\0' * (len(segment) - len(short)) + # Note that the terminator will be copied on the next iteration + last_pos = end_pos + b += b0[last_pos:] + return b + + +def fix_solib_names(strip: Path, solib_groups: dict[str, list[str]], solib_re: re.Pattern[bytes], fn: Path) -> bytes: + b0 = fn.read_bytes() + if b0[:4] == b'\x7fELF': + fd, path = mkstemp() + try: + os.close(fd) + check_call([strip, fn, '-o', path], text=False) + b0 = Path(path).read_bytes() + finally: + os.remove(path) + matches = list(solib_re.finditer(b0)) + if not matches: + return b0 + magic = b0[:4] + if magic in (b'\x7fELF', b'GOBJ'): + is_binary = True + elif magic == b' re.Pattern[bytes]: + return re.compile(b'|'.join(re.escape(name.encode('ascii')) for name in solib_groups.keys())) + + +def test_fix_solib_names(): + solib_groups = { + 'libharfbuzz-gobject.so': ['libharfbuzz-gobject.so', 'libharfbuzz-gobject.so.0'], + 'libharfbuzz.so': ['libharfbuzz.so', 'libharfbuzz.so.0'], + } + solib_re = get_solib_re(solib_groups) + b0 = b"GOBJ\1\2\3libharfbuzz-gobject.so.0,libbla,libharfbuzz.so.0\0FOO" + fd, fn = mkstemp() + try: + os.write(fd, b0) + os.close(fd) + b1 = fix_solib_names(Path('/dev/null'), solib_groups, solib_re, Path(fn)) + finally: + os.remove(fn) + assert b1 == b"GOBJ\1\2\3libharfbuzz-gobject.so,libbla,libharfbuzz.so\0\0\0\0\0FOO"