CMake: Add packaging support (#486)

.github/workflows/vcpkg_ci.yml

@@ -31,12 +31,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Install utility tools
         shell: bash
         run: |
           # TODO some of these should probably live in the Docker build image
           apt-get update
-          apt-get install -y pixz xz-utils make
+          apt-get install -y pixz xz-utils make rpm
 
       - name: Build with build script
         shell: bash
@@ -54,6 +56,33 @@ jobs:
           remill-lift-${{ matrix.llvm }} --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000
           remill-lift-${{ matrix.llvm }} --arch aarch64 --ir_out /dev/stdout --address 0x400544 --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6
 
+      - name: Locate the packages
+        id: package_names
+        shell: bash
+        working-directory: remill-build
+        run: |
+          echo ::set-output name=DEB_PACKAGE_PATH::remill-build/$(ls *.deb)
+          echo ::set-output name=RPM_PACKAGE_PATH::remill-build/$(ls *.rpm)
+          echo ::set-output name=TGZ_PACKAGE_PATH::remill-build/$(ls *.tar.gz)
+
+      - name: Store the DEB package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_deb_package
+          path: ${{ steps.package_names.outputs.DEB_PACKAGE_PATH }}
+
+      - name: Store the RPM package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_rpm_package
+          path: ${{ steps.package_names.outputs.RPM_PACKAGE_PATH }}
+
+      - name: Store the TGZ package
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_tgz_package
+          path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}
+
   build_mac:
     strategy:
       fail-fast: false
@@ -69,6 +98,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Build with build script
         shell: bash
         run: ./scripts/build.sh --llvm-version ${{ matrix.llvm }}
@@ -84,3 +115,16 @@ jobs:
         run: |
           remill-lift-${{ matrix.llvm }} --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000
           remill-lift-${{ matrix.llvm }} --arch aarch64 --ir_out /dev/stdout --address 0x400544 --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6
+
+      - name: Locate the packages
+        id: package_names
+        shell: bash
+        working-directory: remill-build
+        run: |
+          echo ::set-output name=TGZ_PACKAGE_PATH::remill-build/$(ls *.tar.gz)
+
+      - name: Store the TGZ package
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-11.0_llvm${{ matrix.llvm }}_tgz_package
+          path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}

distro_packages/ArchLinux/.gitignore

@@ -1,19 +0,0 @@
-# Copyright (c) 2017 Trail of Bits, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-pkg
-remill
-src
-*.pkg.tar.xz
-

distro_packages/ArchLinux/PKGBUILD

@@ -1,61 +0,0 @@
-pkgname=remill-git
-pkgver=1
-pkgrel=1
-arch=('x86_64')
-
-pkgdesc="Library for lifting of x86, amd64, and aarch64 machine code to LLVM bitcode"
-url="https://github.com/lifting-bits/remill"
-license=('Apache')
-
-depends=('clang39' 'llvm39' 'llvm39-libs')
-makedepends=('git' 'curl' 'patchelf' 'cmake')
-
-replaces=('remill')
-provides=('remill')
-conflicts=('remill')
-
-source=("${pkgname}::git+https://github.com/lifting-bits/remill.git")
-sha1sums=('SKIP')
-
-pkgver() {
-  cd "${srcdir}/${pkgname}"
-  git log -1 --date=format:%Y%m%d --pretty=format:%ad_%h
-}
-
-build() {
-  local clang_version=`clang --version | head -n 1 | awk '{ print $3 }' | tr -d '.' | cut -c -2`
-  local cxxcommon_tarball_name="libraries-llvm${clang_version}-ubuntu1604-amd64.tar.gz"
-  local cxxcommon_tarball_url="https://s3.amazonaws.com/cxx-common/${cxxcommon_tarball_name}"
-
-  if [ ! -f "${cxxcommon_tarball_name}" ] ; then
-    echo "Dowloading: ${cxxcommon_tarball_url}"
-    curl "${cxxcommon_tarball_url}" -O
-  fi
-
-  if [ ! -d "libraries" ] ; then
-    tar xzf "${cxxcommon_tarball_name}"
-
-    # use the system llvm/clang packages
-    rm -rf "libraries/llvm"
-    sed -i '/llvm/d' "libraries/cmake_modules/repository.cmake"
-  fi
-
-  export CXX=clang++
-  export CC=clang
-  export TRAILOFBITS_LIBRARIES="$(realpath libraries)"
-
-  if [ -d build ] ; then
-    rm -rf build
-  fi
-
-  mkdir build
-  cd build
-
-  cmake -DCMAKE_C_COMPILER="${CC}" -DCMAKE_CXX_COMPILER="${CXX}" -DCMAKE_INSTALL_PREFIX="/usr" -DCMAKE_VERBOSE_MAKEFILE=True "${srcdir}/${pkgname}"
-  make -j `nproc`
-}
-
-package() {
-    cd build
-    make DESTDIR="${pkgdir}" install
-}

packaging/README.md

@@ -0,0 +1,17 @@
+# remill packaging scripts
+
+## How to generate packages
+
+1. Configure and build remill
+2. Set the **DESTDIR** variable to a new folder
+3. Run the packaging script, passing the **DESTDIR** folder
+
+Example:
+
+```sh
+remill_version=$(git describe --always)
+
+cpack -D REMILL_DATA_PATH="/path/to/install/directory" \
+      -R ${remill_version} \
+      --config "packaging/main.cmake"
+```

packaging/cmake/dispatcher.cmake

@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CMAKE_SYSTEM_PROCESSOR}")
+set(CPACK_INSTALLED_DIRECTORIES "${REMILL_DATA_PATH};.")
+
+string(TOLOWER "${CMAKE_SYSTEM_NAME}" system_name)
+if(system_name STREQUAL "darwin")
+  set(system_name "macos")
+endif()
+
+set(common_include "${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/common.cmake")
+if(EXISTS "${common_include}")
+  include("${common_include}")
+endif()
+
+string(TOLOWER "${CPACK_GENERATOR}" cpack_generator)
+include("${CMAKE_CURRENT_LIST_DIR}/system/${system_name}/generators/${cpack_generator}.cmake")

packaging/cmake/system/linux/generators/deb.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_DEBIAN_PACKAGE_PRIORITY "extra")
+set(CPACK_DEBIAN_PACKAGE_SECTION "default")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "${CPACK_PACKAGE_HOMEPAGE_URL}")
+

packaging/cmake/system/linux/generators/rpm.cmake

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_RPM_PACKAGE_RELEASE "${CPACK_PACKAGE_VERSION}")
+set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
+set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}")
+set(CPACK_RPM_PACKAGE_GROUP "default")
+set(CPACK_RPM_PACKAGE_LICENSE "GNU Affero General Public License v3.0")

packaging/cmake/system/linux/generators/tgz.cmake

@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_SET_DESTDIR ON)
+set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)

packaging/cmake/system/macos/generators/tgz.cmake

@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+set(CPACK_SET_DESTDIR ON)
+set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)

packaging/main.cmake

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
+  set(CPACK_GENERATOR "TGZ;DEB;RPM")
+
+elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
+  set(CPACK_GENERATOR "TGZ")
+endif()
+
+if(REMILL_DATA_PATH STREQUAL "")
+  message(FATAL_ERROR "The REMILL_DATA_PATH variable was not set")
+endif()
+
+if(REMILL_PACKAGE_VERSION STREQUAL "")
+  message(FATAL_ERROR "The REMILL_PACKAGE_VERSION variable was not set")
+endif()
+
+set(CPACK_PROJECT_CONFIG_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/dispatcher.cmake")
+
+set(CPACK_PACKAGE_DESCRIPTION "Library for lifting of x86, amd64, and aarch64 machine code to LLVM bitcode")
+set(CPACK_PACKAGE_NAME "remill")
+set(CPACK_PACKAGE_VENDOR "Trail of Bits")
+set(CPACK_PACKAGE_CONTACT "opensource@trailofbits.com")
+set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/lifting-bits/remill")

scripts/build.sh

@@ -257,6 +257,34 @@ function Build
   return $?
 }
 
+# Create the packages
+function Package
+{
+  remill_tag=$(cd "${SRC_DIR}" && git describe --tags --always --abbrev=0)
+  remill_commit=$(cd "${SRC_DIR}" && git rev-parse HEAD | cut -c1-7)
+  remill_version="${remill_tag:1}.${remill_commit}"
+
+  (
+    set -x
+
+    if [[ -d "install" ]]; then
+      rm -rf "install"
+    fi
+
+    mkdir "install"
+    export DESTDIR="$(pwd)/install"
+
+    cmake --build . \
+      --target install
+
+    cpack -D REMILL_DATA_PATH="${DESTDIR}" \
+      -R ${remill_version} \
+      --config "${SRC_DIR}/packaging/main.cmake"
+  ) || return $?
+
+  return $?
+}
+
 # Get a LLVM version name for the build. This is used to find the version of
 # cxx-common to download.
 function GetLLVMVersion
@@ -385,7 +413,7 @@ function main
   mkdir -p "${BUILD_DIR}"
   cd "${BUILD_DIR}" || exit 1
 
-  if ! (DownloadLibraries && Configure && Build); then
+  if ! (DownloadLibraries && Configure && Build && Package); then
     echo "[x] Build aborted."
     exit 1
   fi