From 85306e27d6858edf441614d24f4d6296519909d7 Mon Sep 17 00:00:00 2001
From: happysalada <raphael@megzari.com>
Date: Sun, 28 Jan 2024 16:14:51 +0100
Subject: [PATCH 1/2] python311Packages.vllm: init at v0.3.1

---
 .../python-modules/vllm/default.nix           | 139 ++++++++++++++++++
 pkgs/top-level/python-packages.nix            |   2 +
 2 files changed, 141 insertions(+)
 create mode 100644 pkgs/development/python-modules/vllm/default.nix

diff --git a/pkgs/development/python-modules/vllm/default.nix b/pkgs/development/python-modules/vllm/default.nix
new file mode 100644
index 000000000000000..59d5b560d7b0049
--- /dev/null
+++ b/pkgs/development/python-modules/vllm/default.nix
@@ -0,0 +1,139 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, fetchpatch
+, which
+, ninja
+, packaging
+, setuptools
+, torch
+, wheel
+, psutil
+, ray
+, pandas
+, pyarrow
+, sentencepiece
+, numpy
+, transformers
+, xformers
+, fastapi
+, uvicorn
+, pydantic
+, aioprometheus
+, pynvml
+, cupy
+, writeShellScript
+
+, config
+
+, cudaSupport ? config.cudaSupport
+, cudaPackages ? {}
+
+, rocmSupport ? config.rocmSupport
+, rocmPackages ? {}
+, gpuTargets ? []
+}:
+
+buildPythonPackage rec {
+  pname = "vllm";
+  version = "0.3.1";
+  format = "pyproject";
+
+  src = fetchFromGitHub {
+    owner = "vllm-project";
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-hfd4ScU0mkZ7z4+w08BUA1K9bPXSiFThfiO+Ll2MTtg=";
+  };
+
+  # Otherwise it tries to enumerate host supported ROCM gfx archs, and that is not possible due to sandboxing.
+  PYTORCH_ROCM_ARCH = lib.optionalString rocmSupport (lib.strings.concatStringsSep ";" rocmPackages.clr.gpuTargets);
+
+  # xformers 0.0.23.post1 github release specifies its version as 0.0.24
+  #
+  # cupy-cuda12x is the same wheel as cupy, but built with cuda dependencies, we already have it set up
+  # like that in nixpkgs. Version upgrade is due to upstream shenanigans
+  # https://github.com/vllm-project/vllm/pull/2845/commits/34a0ad7f9bb7880c0daa2992d700df3e01e91363
+  #
+  # hipcc --version works badly on NixOS due to unresolved paths.
+  postPatch = ''
+    substituteInPlace requirements.txt \
+      --replace "xformers == 0.0.23.post1" "xformers == 0.0.24"
+    substituteInPlace requirements.txt \
+      --replace "cupy-cuda12x == 12.1.0" "cupy == 12.3.0"
+    substituteInPlace requirements-build.txt \
+      --replace "torch==2.1.2" "torch == 2.2.0"
+    substituteInPlace pyproject.toml \
+      --replace "torch == 2.1.2" "torch == 2.2.0"
+    substituteInPlace requirements.txt \
+      --replace "torch == 2.1.2" "torch == 2.2.0"
+  '' + lib.optionalString rocmSupport ''
+    substituteInPlace setup.py \
+      --replace "'hipcc', '--version'" "'${writeShellScript "hipcc-version-stub" "echo HIP version: 0.0"}'"
+  '';
+
+  preBuild = lib.optionalString cudaSupport ''
+    export CUDA_HOME=${cudaPackages.cuda_nvcc}
+  ''
+  + lib.optionalString rocmSupport ''
+    export ROCM_HOME=${rocmPackages.clr}
+    export PATH=$PATH:${rocmPackages.hipcc}
+  '';
+
+  nativeBuildInputs = [
+    ninja
+    packaging
+    setuptools
+    torch
+    wheel
+    which
+  ] ++ lib.optionals rocmSupport [
+    rocmPackages.hipcc
+  ];
+
+  buildInputs = (lib.optionals cudaSupport (with cudaPackages; [
+    cuda_cudart # cuda_runtime.h, -lcudart
+    cuda_cccl.dev # <thrust/*>
+    libcusparse.dev # cusparse.h
+    libcublas.dev # cublas_v2.h
+    libcusolver # cusolverDn.h
+  ])) ++ (lib.optionals rocmSupport (with rocmPackages; [
+    clr
+    rocthrust
+    rocprim
+    hipsparse
+    hipblas
+  ]));
+
+  propagatedBuildInputs = [
+    psutil
+    ray
+    pandas
+    pyarrow
+    sentencepiece
+    numpy
+    torch
+    transformers
+    xformers
+    fastapi
+    uvicorn
+    pydantic
+    aioprometheus
+  ] ++ uvicorn.optional-dependencies.standard
+    ++ aioprometheus.optional-dependencies.starlette
+    ++ lib.optionals cudaSupport [
+      pynvml
+      cupy
+    ];
+
+  pythonImportsCheck = [ "vllm" ];
+
+  meta = with lib; {
+    description = "A high-throughput and memory-efficient inference and serving engine for LLMs";
+    changelog = "https://github.com/vllm-project/vllm/releases/tag/v${version}";
+    homepage = "https://github.com/vllm-project/vllm";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ happysalada ];
+    broken = !cudaSupport && !rocmSupport;
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index daf59d7beaebe35..00f02e31c768990 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -16024,6 +16024,8 @@ self: super: with self; {
 
   viv-utils = callPackage ../development/python-modules/viv-utils { };
 
+  vllm = callPackage ../development/python-modules/vllm { };
+
   vmprof = callPackage ../development/python-modules/vmprof { };
 
   vncdo = callPackage ../development/python-modules/vncdo { };

From dac6982f9125e47da274f5827cd46228252157d8 Mon Sep 17 00:00:00 2001
From: Yaroslav Bolyukin <iam@lach.pw>
Date: Sat, 17 Feb 2024 14:20:06 +0100
Subject: [PATCH 2/2] python3Packages.vllm: add lach to maintainers

---
 pkgs/development/python-modules/vllm/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/python-modules/vllm/default.nix b/pkgs/development/python-modules/vllm/default.nix
index 59d5b560d7b0049..e8127c4caf8bb49 100644
--- a/pkgs/development/python-modules/vllm/default.nix
+++ b/pkgs/development/python-modules/vllm/default.nix
@@ -133,7 +133,7 @@ buildPythonPackage rec {
     changelog = "https://github.com/vllm-project/vllm/releases/tag/v${version}";
     homepage = "https://github.com/vllm-project/vllm";
     license = licenses.asl20;
-    maintainers = with maintainers; [ happysalada ];
+    maintainers = with maintainers; [ happysalada lach ];
     broken = !cudaSupport && !rocmSupport;
   };
 }