Arch Linux User Repository

I've created a ctranslate2 package, please test.

patch to fix git and build directory

--- PKGBUILD    2023-11-07 20:26:55.090501855 +0200
+++ PKGBUILD.NEW    2023-11-07 20:31:56.665787121 +0200
@@ -3,7 +3,7 @@
 pkgbase=ctranslate2-git
 pkgname=(ctranslate2-git
          python-ctranslate2-git)
-pkgver=1703.77a48d43
+pkgver=2110.f92a8a27
 pkgrel=1
 pkgdesc='Fast inference engine for OpenNMT models'
 arch=('x86_64' 'i686')
@@ -45,7 +45,18 @@
     git config submodule.third_party/thrust.url "$srcdir/thrust"
     git config submodule.third_party/googletest.url "$srcdir/googletest"
     git config submodule.third_party/cpu_features.url "$srcdir/cpu_features"
+
+    for i in cpu_features  ctranslate2  cub  cxxopts  googletest  thrust
+        do cd ${srcdir}/$i
+        git -c protocol.file.allow=always submodule update
+        done
+
+    cd ${srcdir}/ctranslate2
     git submodule update
+
+
+
    }

 build() {
@@ -53,6 +64,13 @@
     # CPU dispatch breaks with multiple definitions blah blah
     cmake ../ \
         -DCMAKE_BUILD_TYPE=Release \
+       # to enable features. multiple can enabled at once
+       # cudnn: nvidia neural network . Enabling it currently causing build error
+       # cuda:  nvidia cuda . This can also error out currently
+       # openblass: blass but for cpu
+       # -DWITH_CUDNN=ON \
+       # -DWITH_CUDA=ON \
+       # -DWITH_OPENBLAS=ON \
         -DCMAKE_INSTALL_PREFIX=/usr \
         -DWITH_MKL=False \
         -DENABLE_CPU_DISPATCH=False \
@@ -72,7 +90,7 @@


 package_ctranslate2-git() {
-    cd ${srcdir}/build
+    cd ${srcdir}/ctranslate2/build/
     make DESTDIR="$pkgdir" install
 }

git complains with fatal: transport 'file' not allowed unless you change it to git -c protocol.file.allow=always submodule update.

Missing submodules: third_party/ruy, third_party/ruy/third_party/cpuinfo, third_party/ruy/third_party/googletest, third_party/spdlog, third_party/thrust/dependencies/cub

Removed submodules: third_party/cub

And package() is failing since commit 55b94d3 because the build directory changed. And it installs files that conflict with nlohmann-json.

pkgver() should return an actual version number via git describe.

Since git 2.38. the git config calls now need the --global option.

Line 75 does not work. And generally makepkg will always cd into $srcdir ,so you could shorten the cd-lines.

cd ctranslate2/build

The python part should be split out of the package and put into a separate PKGBUILD depending on this one.

# Maintainer: Martin Sandsmark <martin.sandsmark@kde.org>

pkgname=ctranslate2-git
pkgver=1510.148c52c
pkgrel=1
pkgdesc='Fast inference engine for OpenNMT models'
arch=('x86_64' 'i686')
url='https://github.com/OpenNMT/ctranslate2'
license=('GPL3')
makedepends=('git' 'cmake' 'pybind11')
conflicts=(ctranslate2)
provides=(ctranslate2)
source=(
        'git+https://github.com/OpenNMT/ctranslate2.git'

        'git+https://github.com/jarro2783/cxxopts.git'
        'git+https://github.com/NVlabs/cub.git'
        'git+https://github.com/thrust/thrust.git'
        'git+https://github.com/google/googletest.git'
        'git+https://github.com/google/cpu_features.git'
)
md5sums=('SKIP'
         'SKIP'
         'SKIP'
         'SKIP'
         'SKIP'
         'SKIP'
         )

pkgver() {
    cd ctranslate2
    printf "%s.%s" "$(git rev-list --count HEAD)" "$(git rev-parse --short HEAD)"
}

prepare() {
    mkdir -p build
    cd ctranslate2
    git submodule init
    git config submodule.third_party/cxxopts.url "$srcdir/cxxopts"
    git config submodule.third_party/cub.url "$srcdir/cub"
    git config submodule.third_party/thrust.url "$srcdir/thrust"
    git config submodule.third_party/googletest.url "$srcdir/googletest"
    git config submodule.third_party/cpu_features.url "$srcdir/cpu_features"
    git submodule update
   }

build() {
    cd build

    # CPU dispatch breaks with multiple definitions blah blah
    cmake ../ctranslate2 \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_INSTALL_PREFIX=/usr \
        -DWITH_MKL=False \
        -DENABLE_CPU_DISPATCH=False \
        -DOPENMP_RUNTIME=COMP

    # Not enabled/added as deps:
    #   WITH_MKL (seems to be incompatible with my version, at least)
    #   WITH_DNNL
    #   WITH_OPENBLAS
    #   WITH_CUDA

    make
}


package() {
    cd build
    make DESTDIR="$pkgdir" install

}

and something like

# Maintainer: Martin Sandsmark <martin.sandsmark@kde.org>

pkgname=python-ctranslate2-git
pkgver=1510.148c52c
pkgrel=1
pkgdesc='Fast inference engine for OpenNMT models'
arch=('x86_64' 'i686')
url='https://github.com/OpenNMT/ctranslate2'
license=('GPL3')
makedepends=('git' 'cmake' 'pybind11')
conflicts=(python-ctranslate2)
provides=(python-ctranslate2)
source=('git+https://github.com/OpenNMT/ctranslate2.git')
md5sums=('SKIP')

pkgver() {
  cd ctranslate2
  printf "%s.%s" "$(git rev-list --count HEAD)" "$(git rev-parse --short HEAD)"
}

package() {
  cd ctranslate2/python
  python setup.py install --root="$pkgdir" --optimize=1
}

It's not building properly

copying ctranslate2/specs/model_spec.py -> build/lib.linux-x86_64-3.10/ctranslate2/specs
running build_ext
building 'ctranslate2.translator' extension
creating build/temp.linux-x86_64-3.10
gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -march=x86-64 -mtune=generic -O3 -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -ffat-lto-objects -march=x86-64 -mtune=generic -O3 -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -march=x86-64 -mtune=generic -O3 -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -fPIC -I/usr/lib/python3.10/site-packages/pybind11/include -I/usr/include/python3.10 -c translator.cc -o build/temp.linux-x86_64-3.10/translator.o -std=c++17
translator.cc:11:10: fatal error: ctranslate2/translator_pool.h: No such file or directory
   11 | #include <ctranslate2/translator_pool.h>
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.
error: command '/opt/cuda/bin/gcc' failed with exit code 1
==> ERROR: A failure occurred in build().
    Aborting...
 -> error making: ctranslate2-git

It's also missing a make dependency on python-setuptools.

I think the fundamental issue here is that setup.py is supposed to run after installing the main stuff built by cmake. I can get the same thing to happen if I build from source without the PKGBUILD and I don't run make install before setup.py

I was able to get it to build by making the following change to the PKGBUILD, but I don't know if it will work at runtime because it changes what path setup.py uses for the library.

diff --git a/PKGBUILD b/PKGBUILD
index d53c37c..9dcbff1 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,14 +1,14 @@
 # Maintainer: Martin Sandsmark <martin.sandsmark@kde.org>

 pkgname=ctranslate2-git
-pkgver=1235.6d8b6f9
+pkgver=1476.bddebaa
 pkgrel=1
 pkgdesc='Fast inference engine for OpenNMT models'
 arch=('x86_64' 'i686')
 url='https://github.com/OpenNMT/ctranslate2'
 license=('GPL3')
 depends=('python')
-makedepends=('git' 'cmake' 'pybind11')
+makedepends=('git' 'cmake' 'pybind11' 'python-setuptools')
 conflicts=(ctranslate2)
 provides=(ctranslate2)
 source=(
@@ -38,6 +38,7 @@ pkgver() {

 prepare() {
     mkdir -p build
+    mkdir -p lib
     cd ctranslate2
     git submodule init
     git config submodule.third_party/cxxopts.url "$srcdir/cxxopts"
@@ -67,8 +68,10 @@ build() {

     make

+    cp *.so "$srcdir"/ctranslate2/lib
+
     cd "$srcdir"/ctranslate2/python
-    python setup.py build
+    CTRANSLATE2_ROOT="$srcdir"/ctranslate2 python setup.py build
 }

translator.cc:11:10: fatal error: ctranslate2/translator_pool.h: No such file or directory
   11 | #include <ctranslate2/translator_pool.h>
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.
error: command '/usr/bin/gcc' failed with exit code 1
==> ERROR: A failure occurred in build().
    Aborting...
error: failed to build 'ctranslate2-git-1235.6d8b6f9-1': 
error: packages failed to build: ctranslate2-git-1235.6d8b6f9-1

Surely there's a cleaner solution than that. I tried to figure out moving it into the build dir but that didn't work.