\documentclass{beamer}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{qrcode}
\usepackage[normalem]{ulem}

\title{Hot cross builds \\ Cross-compilation in pkgsrc}
\author{Taylor R Campbell \\
  \texttt{riastradh@NetBSD.org}}
\date{BSDCan 2024 \\
  Ottawa, Canada \\
  May 31, 2024}

\begin{document}

\frame{\titlepage}

\begin{frame}
  \frametitle{Hot cross builds: cross-compilation in pkgsrc}

  \centering

  \url{https://www.NetBSD.org/gallery/presentations/riastradh/bsdcan2024/pkgcross.pdf}

  \vspace{\baselineskip}

  \qrcode[height=2in]{https://www.NetBSD.org/gallery/presentations/riastradh/bsdcan2024/pkgcross.pdf}
\end{frame}

\begin{frame}
  \frametitle{pkgsrc: portable package build system}

  \begin{itemize}
    \item \texttt{https://pkgsrc.org/}

    \item Framework for building third-party software on Unix-like
       operating systems.

    \item ${>}26{,}000$ packages.

    \item Actively supported platforms:
      \begin{itemize}
        \item NetBSD (first platform, based on mid-'90s FreeBSD ports)
        \item Solaris/SmartOS/illumos
        \item Linux
        \item macOS
      \end{itemize}

    \item Other platforms with some support:
      \begin{itemize}
        \item FreeBSD/OpenBSD/DragonflyBSD/MidnightBSD
        \item MINIX 3
        \item SCO OpenServer/UnixWare
        \item HP-UX
        \item QNX
      \end{itemize}

    \item Works unprivileged, so you can develop in your home
       directory on a server you don't administer.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Anatomy of a pkgsrc package}

  \begin{itemize}
    \item \texttt{DESCR} -- Human-readable description.
    \item \texttt{Makefile} --
      Machine-readable description.
      \begin{itemize}
        \item Tells where to download source code.
        \item Rules for how to configure, build, install.
        \item Etc.
      \end{itemize}
    \item \texttt{distinfo} --
      Names, sizes, and hashes of source distribution.
      Provides cryptographic integrity check.
    \item \texttt{PLIST} --
      Packing list: lists files installed by package.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{pkgsrc example: \texttt{security/nettle}, part 1}

  \begin{center}
    \small
\begin{verbatim}
# $NetBSD: pkgcross.tex,v 1.2 2024/05/31 12:12:59 riastradh Exp $

DISTNAME=       nettle-3.9.1
CATEGORIES=     security
MASTER_SITES=   http://www.lysator.liu.se/~nisse/archive/
MASTER_SITES+=  ftp://ftp.lysator.liu.se/pub/security/lsh/

MAINTAINER=     pkgsrc-users@NetBSD.org
HOMEPAGE=       https://www.lysator.liu.se/~nisse/nettle/
COMMENT=        Cryptographic library
LICENSE=        gnu-lgpl-v2.1

USE_LANGUAGES=          c c99
USE_LIBTOOL=            yes
USE_TOOLS+=             gm4 gmake
GNU_CONFIGURE=          yes
SET_LIBDIR=             yes
CONFIGURE_ARGS+=        --disable-openssl
CONFIGURE_ARGS+=        --disable-shared
\end{verbatim}
  \end{center}
\end{frame}

\begin{frame}[fragile]
  \frametitle{pkgsrc example: \texttt{security/nettle}, part 2}

  \begin{center}
    \small
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if ${USE_CROSS_COMPILE:tl} == "yes"
CONFIGURE_ENV+=         CC_FOR_BUILD=${NATIVE_CC:Q}
.endif

INFO_FILES=             yes
TEST_TARGET=            check
PKGCONFIG_OVERRIDE=     hogweed.pc.in
PKGCONFIG_OVERRIDE+=    nettle.pc.in

BUILDLINK_API_DEPENDS.gmp+=     gmp>=6.0
.include "../../devel/gmp/buildlink3.mk"
.include "../../mk/bsd.pkg.mk"
\end{verbatim}
  \end{center}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Building and installing a
     package\footnote{On NetBSD, can use base system's \texttt{make}, but
      everywhere else we bootstrap \texttt{devel/bmake} for pkgsrc.}}

  \begin{center}
    \footnotesize
\begin{verbatim}
# which socat
socat not found
# cd /usr/pkgsrc/net/socat
# bmake install
=> Bootstrap dependency digest>=20211023: found digest-20220214
=> Fetching socat-1.8.0.0.tar.gz
...
=> Checksum SHA512 OK for socat-1.8.0.0.tar.gz
===> Installing dependencies for socat-1.8.0.0
...
=> Tool dependency checkperms>=1.1: found checkperms-1.12
=> Full dependency readline>=6.0: found readline-8.2nb2
...
=> Creating binary package .../socat-1.8.0.0.tgz
===> Installing binary package of socat-1.8.0.0
# which socat
/usr/pkg/bin/socat
\end{verbatim}
  \end{center}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Binary packages: build once, install many times}

  \begin{itemize}
    \item Building from source is necessary: verify source, audit
       programs, modify, etc.
    \item Building from source is slow: run compiler on lots of source
       code.
    \item Do it once, save the result, install binary packages after.
      \begin{center}
\begin{verbatim}
builder$ cd /home/builder/pkgsrc/net/socat
builder$ bmake package

client# PKG_PATH=/nfs/builder/pkgsrc/packages
client# export PKG_PATH
client# pkg_add socat
client# which socat
/usr/pkg/bin/socat
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Binary package bulk builds}

  \begin{itemize}
    \item NetBSD provides binary packages for NetBSD on many
       architectures\footnote{\texttt{https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/}}.
    \item MNX Cloud provides binary packages for
       SmartOS, macOS, Linux, and
       NetBSD/amd64\footnote{\texttt{https://pkgsrc.smartos.org/}}.
    \item I build binary packages for my own machines.
    \item You can too!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Cross-compiling NetBSD}

  \begin{itemize}
    \item Every NetBSD build is a cross-build.
    \item \texttt{build.sh tools} builds cross-toolchain.
    \item \texttt{build.sh kernel=GENERIC distribution} builds NetBSD
       with the cross-toolchain.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Cross-compiling pkgsrc}

  \begin{itemize}
    \item Use NetBSD \texttt{build.sh tools distribution} to get
       started.\footnote{See \texttt{doc/HOWTO-use-crosscompile} for
         details.}
    \item \texttt{USE\_CROSS\_COMPILE=yes}
    \item \texttt{TOOLDIR=/usr/obj.evbppc/tooldir.NetBSD-10.0-amd64}
    \item \texttt{CROSS\_DESTDIR=/usr/obj.evbppc/destdir.evbppc}
    \item \texttt{CROSS\_MACHINE\_ARCH=powerpc},
       \texttt{CROSS\_OPSYS=NetBSD}, \dots
  \end{itemize}

  \begin{center}
\begin{verbatim}
$ uname -m
amd64
$ cd ~/pkgsrc/net/socat
$ bmake package
...
$ cd ~/pkgsrc/packages.NetBSD-10.0-powerpc/All
$ pkg_info -Q MACHINE_ARCH socat-1.8.0.0.tgz
powerpc
\end{verbatim}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Cross-build in homedir, install systemwide on target}

  \begin{itemize}
    \item \texttt{./bootstrap --prefix /home/builder/pkg --unprivileged
       \dots}
    \item set \texttt{CROSS\_LOCALBASE=/usr/pkg} in mk.conf
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Toolchain wrappers}

  \begin{itemize}
    \item pkgsrc creates symlink farms of toolchain wrappers for build:
      \begin{itemize}
        \item \texttt{cc}, \texttt{ld}, \texttt{as}, \dots
        \item \texttt{powerpc--netbsd-gcc},
           \texttt{powerpc--netbsd-ld},
           \texttt{powerpc--netbsd-as},
           \dots
      \end{itemize}
    \item pkgsrc buildlink3 framework creates symlink farms of
       dependent header files and libraries for build isolation.
    \item Wrappers transform toolchain arguments:
      \begin{itemize}
        \item add \texttt{--sysroot=\$\{CROSS\_DESTDIR\}}
        \item ensure \texttt{-I} (build-time include path) and
           \texttt{-L} (build-time library path) point at
           buildlink3 symlink farms
        \item ensure \texttt{-Wl,-R} (run-time library path) points at
           installation prefix without \texttt{CROSS\_DESTDIR}
        \item replace \texttt{-ldl} by appropriate platform-specific
           dlfcn.h option
        \item other package-specific argument transformations
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Dependencies}

  \begin{itemize}
    \item Some packages \textbf{depend} on other packages:
      \begin{itemize}
        \item \texttt{tor} program needs \texttt{libevent} library at
           run-time
          \begin{itemize}
            \item \texttt{net/tor} (\textbf{run-}) \textbf{depends} on
              \texttt{devel/libevent}
          \end{itemize}
        \item Compiler needs \texttt{event.h} when building
           \texttt{tor} program at compile-time
          \begin{itemize}
            \item \texttt{net/tor} also \textbf{build-depends}
               on \texttt{devel/libevent}
          \end{itemize}
        \item Building \texttt{libxcb} requires running
           \texttt{xsltproc} to turn XML into C header files at
           compile-time
          \begin{itemize}
            \item \texttt{x11/libxcb} \textbf{tool-depends} on
               \texttt{textproc/xsltproc}
          \end{itemize}
        \item {\scriptsize Also \textbf{bootstrap-depends}, like
           tool-depends but for parts of the pkgsrc infrastructure.}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Cross-compiling dependencies}

  \begin{itemize}
    \item Use Intel Xeon to build \texttt{x11/xterm}, run on your
       powerpc-based thin client.
    \item \texttt{x11/xterm} must be \emph{cross-built} for
       \texttt{MACHINE\_ARCH=powerpc}.
    \item \texttt{x11/xterm} depends on \texttt{x11/libxcb}\footnote{Via
       \texttt{x11/libX11}.}.
      \begin{itemize}
        \item \texttt{x11/libxcb} must be \emph{cross-built} for
           \texttt{MACHINE\_ARCH=powerpc}.
      \end{itemize}
   \item \texttt{x11/libxcb} \emph{tool-depends} on
       \texttt{textproc/xsltproc}.
      \begin{itemize}
        \item \texttt{textproc/libxsltproc} must be
           \emph{natively built} for \texttt{MACHINE\_ARCH=x86\_64}.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Build-depends vs tool-depends}

  \begin{itemize}
    \item Both build-depends and tool-depends need to exist at
       build-time.

    \item \emph{Build-depends} are cross-built and installed into
       \texttt{/usr/obj.evbppc/destdir.evbppc/usr/pkg/...}
      \begin{itemize}
        \item Example: C libraries, needed for linker.
      \end{itemize}

    \item \emph{Tool-depends} are natively built and installed into
       \texttt{/home/builder/pkg/...} (\texttt{\$\{TOOLBASE\}})
      \begin{itemize}
        \item Example: \texttt{xsltproc}, cross-compiler.
        \item \texttt{TARGET\_MACHINE\_ARCH}, \texttt{TARGET\_OPSYS},
           \dots, are set to cross-compilation target.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Pointing builds at tool programs in dependencies}

  \begin{itemize}
    \item Package uses glib-mkenums at build-time, how to use it?
      \begin{verbatim}
TOOL_DEPENDS+=          \
    glib2-tools>=0:../../devel/glib2-tools
\end{verbatim}
    \item GNU Autoconf:
      \begin{verbatim}
CONFIGURE_ARGS+=        \
    GLIB_MKENUMS=${TOOLBASE:Q}/bin/glib-mkenums
\end{verbatim}
    \item Meson:
      \begin{verbatim}
MESON_CROSS_BINARIES+=  glib-mkenums
MESON_CROSS_BINARY.glib-mkenums=        \
    ${TOOLBASE}/bin/glib-mkenums
\end{verbatim}
      \item Similarly: Use \texttt{TOOL\_PYTHONBIN} at build-time, but
         bake \texttt{PYTHONBIN} into product for run-time Python.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Meson: pkgsrc creates cross config for you}

  \small
  \begin{verbatim}
[properties]
sys_root = '/usr/obj.evbppc/destdir.evbppc'
[host_machine]
system = 'netbsd'
cpu_family = 'ppc'
cpu = 'powerpc'
endian = 'big'
[binaries]
glib-genmarshal = '/home/builder/pkg/bin/glib-genmarshal'
glib-mkenums = '/home/builder/pkg/bin/glib-mkenums'
\end{verbatim}
\end{frame}

\begin{frame}
  \frametitle{Complications part 1: mixing up build-depends and
     tool-depends}

  \begin{itemize}
    \item Originally, pkgsrc had only build-depends---same as
       tool-depends for native builds.
      \begin{itemize}
        \item \texttt{x11/libxcb} build-depended on
           \texttt{textproc/xsltproc}.
      \end{itemize}
    \item Packages practically never need to set
       \texttt{BUILD\_DEPENDS} directly---only via buildlink3.
    \item Solution: We mass-changed \texttt{BUILD\_DEPENDS} to
       \texttt{TOOL\_DEPENDS} in package makefiles.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Complications part 2: package builds tools internally}

  \begin{itemize}
    \item Some packages depend on external tools like
       \texttt{x11/libxcb} depends on \texttt{textproc/xsltproc}.
    \item Others use internal tools, like \texttt{security/nettle}
       above.
    \item These try to use \texttt{CC}, which may be
       \texttt{powerpc--netbsd-gcc} for cross-compilation.
    \item Can't run the result on x86!
    \item Solution: Set \texttt{CC\_FOR\_BUILD}, maybe patch
       package to use it instead.
      \begin{center}
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if ${USE_CROSS_COMPILE:tl} == "yes"
CONFIGURE_ENV+= CC_FOR_BUILD=${NATIVE_CC:Q}
.endif
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Complications part $2'$: package runs its own build product}

  \begin{itemize}
    \item Some packages want to run a program they also install.
      \begin{itemize}
        \item \texttt{x11/gtk2} calls \texttt{gtk2-update-icon-cache}.
      \end{itemize}
    \item Need both native \emph{and} cross versions of the program!
    \item Solution: Have package tool-depend on itself and pass path to
       the natively built tool in the cross-build:
      \begin{center}
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if ${USE_CROSS_COMPILE:tl} == "yes"
TOOL_DEPENDS+=      ${PKGNAME}:../../${PKGPATH}
UPDATE_ICON_CACHE=  \
    ${TOOLBASE:Q}/bin/gtk2-update-icon-cache
CONFIGURE_ENV+=     \
    GTK2_UPDATE_ICON_CACHE=${UPDATE_ICON_CACHE}
.endif
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Complications part 3: file existence tests}

  \begin{itemize}
    \item Package wants to know whether \texttt{/dev/urandom} will
       exist when run.
    \item Uses GNU Autoconf to ask whether
       \texttt{/dev/urandom} exists \emph{now}, when built.
    \item Build machine and target system may be different!
    \item But we know \texttt{/dev/urandom} will exist.
    \item Solution: Tell configure up front:
      \begin{center}
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if ${USE_CROSS_COMPILE:tl} == "yes"
CONFIGURE_ENV.NetBSD+=  ac_cv_file__dev_urandom=yes
.endif
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Complications part $3'$: file existence tests in pkgsrc}

  \begin{itemize}
    \item From \texttt{x11/libdrm}:
      \begin{center}\small
\begin{verbatim}
.if !exists(/usr/include/sys/atomic.h)
# libdrm won't find system atomic ops, use a package.
.  include "../../devel/libatomic_ops/buildlink3.mk"
.endif
\end{verbatim}
      \end{center}
    \item Solution: Don't look in \texttt{/usr/include} --- look in
       \texttt{/usr/obj.evbppc/destdir.evbppc}:
      \begin{center}\small
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if !exists(${_CROSS_DESTDIR}/usr/include/sys/atomic.h)
# libdrm won't find system atomic ops, use a package.
.  include "../../devel/libatomic_ops/buildlink3.mk"
.endif
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Complications part 4a: configure run-tests}

  \begin{itemize}
    \item Similar to file existence tests.
    \item Program wants to know \texttt{sizeof(long)} at compile-time.
    \item Compiles a test program to print it, runs test program.
    \item Can't do that if building on 64-bit amd64 for 32-bit powerpc!
    \item Solution: Binary search with compile-time assertions using
      cross-compiler.
    \item (Yes, seriously!  GNU Autoconf supports this with
       \texttt{AC\_CHECK\_SIZEOF}.)
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Complications part 4b: configure run-tests}

  \begin{itemize}
    \item Some are harder to replace.
    \item Tell the answers up front, maybe with patches.
    \item From \texttt{shells/zsh}:
      \begin{center}
\begin{verbatim}
.include "../../mk/bsd.prefs.mk"

.if ${USE_CROSS_COMPILE:tl} == "yes"
.if ${OPSYS} == "NetBSD"
CONFIGURE_ENV+= zsh_cv_shared_environ=yes
CONFIGURE_ENV+= zsh_cv_shared_tgetent=yes
CONFIGURE_ENV+= zsh_cv_shared_tigetstr=yes
CONFIGURE_ENV+= zsh_cv_sys_dynamic_execsyms=yes
.endif
.endif
\end{verbatim}
      \end{center}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Complications part 5: problem children}

  \begin{itemize}
    \item Some packages go to great effort to resist
       cross-compilation.
      \begin{itemize}
        \item Perl
        \item \only<1>{Python}\only<2->{\sout{Python}
          \only<3->{(much better since 3.10)}}
        \item gobject-introspection
      \end{itemize}
    \item Workaround: just build on your powerpc thin client and ship
       binary packages back to x86 build machine to continue.
    \item (Solution: Chainsaws and rototillers.
      Fix the build systems!\footnote{It can be done for Perl: OpenWrt
         does it.
        If you would like to help adapt their approach to pkgsrc, talk
         to me!})
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Related work}

  \begin{itemize}
    \item OpenWrt: cross-compiled packages for Linux-based network
       appliances.
      \begin{itemize}
        \item Linux-only.
        \item Not general-purpose package system.
        \item Much smaller than pkgsrc.
      \end{itemize}

    \item \texttt{distcc}: run pkgsrc on thin client, run compiler
       remotely on x86 build machine.
      \begin{itemize}
        \item Complex to set up: many moving parts (literally).
        \item Hard to parallelize.
        \item Compiler is a big part but not all of run-time---make(1)
           is a big part of pkgsrc cost.
      \end{itemize}

    \item FreeBSD ports: run native compiler in user-mode emulator.
      \begin{itemize}
        \item Many moving parts (figuratively).
        \item Emulators are slow.
        \item Less clean separation between host and target.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{\only<1>{Future}\only<2->{\sout{Future} Past} work}

  \only<2->{(since AsiaBSDcon 2015)}
\end{frame}

\begin{frame}
  \frametitle{\sout{Future} Past work}
  \begin{itemize}
    \item Cross-OS compilation.
      Use SmartOS x86 cloud cluster to build for
       \texttt{MACHINE\_PLATFORM=NetBSD-7.0-powerpc}.
      \begin{itemize}
        \item<2-> Set both \texttt{CROSS\_MACHINE\_ARCH} and
           \texttt{CROSS\_OPSYS} in mk.conf.
        \item<3-> Still to fix: \texttt{USE\_TOOLS+= \dots:run}.
          pkgsrc doesn't distinguish host OS from target OS in
           \texttt{USE\_TOOLS}.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{\sout{Future} Past work}

  \begin{itemize}
    \item User interface improvements.
      \begin{itemize}
        \item<2-> \only<2>{Can't do \texttt{bmake package
            MACHINE\_ARCH=powerpc} for stupid reasons.}
          \only<3->{\sout{Can't do \texttt{bmake package
            MACHINE\_ARCH=powerpc} for stupid reasons.}}
          \begin{itemize}
            \item<3->
              \texttt{bmake package CROSS\_MACHINE\_ARCH=powerpc}
          \end{itemize}
        \item<2-> \only<2>{Setting up cross-compiling requires a manual
           step to work around broken GNU \texttt{libtool}.}
          \only<3->{\sout{Setting up cross-compiling requires a manual
           step to work around broken GNU \texttt{libtool}.}}
          \begin{itemize}
            \item<3-> Bug fixed!
          \end{itemize}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{\sout{Future} \only<1>{Past}\only<2->{\sout{Past} Future} work}

  \begin{itemize}
    \item Bulk builds.
      \begin{itemize}
        \item \texttt{pbulk} doesn't understand build-depends vs
           tool-depends.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{\sout{Future} Past work}

  \begin{itemize}
    \item Unprivileged builds for privileged installs.
      \begin{itemize}
        \item \only<1>{Native and cross packages must both point at
           \texttt{/usr/pkg}.}
          \only<2->{\sout{Native and cross packages must both point at
           \texttt{/usr/pkg}.}}
          \begin{itemize}
            \item<2->
              \texttt{LOCALBASE=/home/builder/pkg} and
              \texttt{CROSS\_LOCALBASE=/usr/pkg} in the same mk.conf.
          \end{itemize}
        \item (Unprivileged builds for unprivileged installs work
           fine---not a problem with privileges, just with different
           paths.)
        \item<3-> Some remaining issues: \texttt{chown} tool, suid
           executables.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Now get cross-building!}

  Questions?
\end{frame}

\end{document}